Add Suricata module to Filebeat

CHANGELOG-developer.asciidoc

@@ -30,6 +30,7 @@ The list below covers the major changes between 6.3.0 and master only.
   has a magefile.go with a fields target. The `FIELDS_FILE_PATH` make variable is no longer
   used because the value is specified in magefile.go. {pull}7670[7670]
 - Outputs must implement String. {pull}6404[6404]
+- Renamed `-beat-name` CLI option used in `kibana_index_pattern.go` to `-beat` for consistency with other scripts in `dev-tools/cmd`. {pull}8615[8615]
 
 ==== Bugfixes
 
@@ -54,3 +55,9 @@ The list below covers the major changes between 6.3.0 and master only.
 - Set current year in generator templates. {pull}8396[8396]
 - You can now override default settings of libbeat by using instance.Settings. {pull}8449[8449]
 - Add `-space-id` option to `export_dashboards.go` script to support Kibana Spaces {pull}7942[7942]
+- Add `-name` option to `asset.go` script to explicitly name the asset rather than using its filename. {pull}8693[8693]
+- Add `-out` option to `kibana_index_pattern.go` to control the output dir to make it possible to write the generated output to `build/kibana` instead of `_meta/kibana.generated` (but the output dir remains unchanged at this point). {pull}8615[8615]
+- Add `module_fields.go` for generated `fields.go` files for modules. {pull}8615[8615]
+- Add `mage.GenerateModuleReferenceConfig` for generating reference config files that include configuration sections from the module directory. {pull}8615[8615]
+- Add `mage.GenerateFieldsGo` for generating fields.go files. {pull}8615[8615]
+- Add `mage.KibanaDashboards` for collecting Kibana dashboards and generating index patterns. {pull}8615[8615]

CHANGELOG.asciidoc

@@ -138,6 +138,7 @@ https://github.com/elastic/beats/compare/v6.4.0...master[Check the HEAD diff]
 - Keep unparsed user agent information in user_agent.original. {pull}8537[8537]
 - Better tracking of number of open file descriptors. {pull}7986[7986]
 - Added default and TCP parsing formats to HAproxy module {issue}8311[8311] {pull}8637[8637]
+- Add Suricata IDS/IDP/NSM module. {issue}8153[8153] {pull}8693[8693]
 - Allow to force CRI format parsing for better performance {pull}8424[8424]
 
 *Heartbeat*

dev-tools/cmd/asset/asset.go

@@ -37,6 +37,7 @@ var (
 	pkg     string
 	input   string
 	output  string
+	name    string
 	license = "ASL2"
 )
 
@@ -45,6 +46,7 @@ func init() {
 	flag.StringVar(&input, "in", "-", "Source of input. \"-\" means reading from stdin")
 	flag.StringVar(&output, "out", "-", "Output path. \"-\" means writing to stdout")
 	flag.StringVar(&license, "license", "ASL2", "License header for generated file.")
+	flag.StringVar(&name, "name", "", "Asset name")
 }
 
 func main() {
@@ -94,9 +96,12 @@ func main() {
 		os.Exit(1)
 	}
 	var buf bytes.Buffer
+	if name == "" {
+		name = file
+	}
 	asset.Template.Execute(&buf, asset.Data{
 		Beat:    beatName,
-		Name:    file,
+		Name:    name,
 		Data:    encData,
 		License: licenseHeader,
 		Package: pkg,

dev-tools/cmd/kibana_index_pattern/kibana_index_pattern.go

@@ -19,51 +19,73 @@ package main
 
 import (
 	"flag"
-	"fmt"
-	"os"
+	"log"
+	"path/filepath"
 
 	"github.com/elastic/beats/libbeat/common"
 	"github.com/elastic/beats/libbeat/kibana"
 	"github.com/elastic/beats/libbeat/version"
 )
 
+var usageText = `
+Usage: kibana_index_pattern [flags]
+  kibana_index_pattern generates Kibana index patterns from the Beat's
+  fields.yml file. It will create a index pattern file that is usable with both
+  Kibana 5.x and 6.x.
+Options:
+`[1:]
+
+var (
+	beatName       string
+	beatVersion    string
+	indexPattern   string
+	fieldsYAMLFile string
+	outputDir      string
+)
+
+func init() {
+	flag.StringVar(&beatName, "beat", "", "Name of the beat. (Required)")
+	flag.StringVar(&beatVersion, "version", version.GetDefaultVersion(), "Beat version. (Required)")
+	flag.StringVar(&indexPattern, "index", "", "Kibana index pattern. (Required)")
+	flag.StringVar(&fieldsYAMLFile, "fields", "fields.yml", "fields.yml file containing all fields used by the Beat.")
+	flag.StringVar(&outputDir, "out", "build/kibana", "Output dir.")
+}
+
 func main() {
-	index := flag.String("index", "", "The name of the index pattern. (required)")
-	beatName := flag.String("beat-name", "", "The name of the beat. (required)")
-	beatDir := flag.String("beat-dir", "", "The local beat directory. (required)")
-	beatVersion := flag.String("version", version.GetDefaultVersion(), "The beat version.")
+	log.SetFlags(0)
 	flag.Parse()
 
-	if *index == "" {
-		fmt.Fprint(os.Stderr, "The name of the index pattern must be set.")
-		os.Exit(1)
+	if beatName == "" {
+		log.Fatal("Name of the Beat must be set (-beat).")
 	}
 
-	if *beatName == "" {
-		fmt.Fprint(os.Stderr, "The name of the beat must be set.")
-		os.Exit(1)
+	if beatVersion == "" {
+		log.Fatal("Beat version must be set (-version).")
 	}
 
-	if *beatDir == "" {
-		fmt.Fprint(os.Stderr, "The beat directory must be set.")
-		os.Exit(1)
+	if indexPattern == "" {
+		log.Fatal("Index pattern must be set (-index).")
 	}
 
 	version5, _ := common.NewVersion("5.0.0")
 	version6, _ := common.NewVersion("6.0.0")
-	versions := []*common.Version{version5, version6}
+	versions := []common.Version{*version5, *version6}
 	for _, version := range versions {
+		indexPattern, err := kibana.NewGenerator(indexPattern, beatName, fieldsYAMLFile, outputDir, beatVersion, version)
+		if err != nil {
+			log.Fatal(err)
+		}
 
-		indexPatternGenerator, err := kibana.NewGenerator(*index, *beatName, *beatDir, *beatVersion, *version)
+		file, err := indexPattern.Generate()
 		if err != nil {
-			fmt.Fprintf(os.Stderr, err.Error())
-			os.Exit(1)
+			log.Fatal(err)
 		}
-		pattern, err := indexPatternGenerator.Generate()
+
+		// Log output file location.
+		absFile, err := filepath.Abs(file)
 		if err != nil {
-			fmt.Fprintf(os.Stderr, err.Error())
-			os.Exit(1)
+			absFile = file
 		}
-		fmt.Fprintf(os.Stdout, "-- The index pattern was created under %v\n", pattern)
+		log.Printf(">> The index pattern was created under %v", absFile)
 	}
 }

dev-tools/cmd/module_fields/main.go

@@ -0,0 +1,118 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"go/format"
+	"io/ioutil"
+	"log"
+	"os"
+	"path"
+
+	"github.com/elastic/beats/libbeat/asset"
+	"github.com/elastic/beats/libbeat/generator/fields"
+	"github.com/elastic/beats/licenses"
+)
+
+var usageText = `
+Usage: module_fields [flags] [module-dir]
+  module_fields generates a fields.go file containing a copy of the module's
+  field.yml data in a format that can be embedded in Beat's binary. module-dir
+  should be the directory containing modules (e.g. filebeat/module).
+Options:
+`[1:]
+
+var (
+	beatName string
+	license  string
+)
+
+func init() {
+	flag.StringVar(&beatName, "beat", "", "Name of the beat. (Required)")
+	flag.StringVar(&license, "license", "ASL2", "License header for generated file.")
+	flag.Usage = usageFlag
+}
+
+func main() {
+	log.SetFlags(0)
+	flag.Parse()
+
+	if beatName == "" {
+		log.Fatal("You must use -beat to specify the beat name.")
+	}
+
+	license, err := licenses.Find(license)
+	if err != nil {
+		log.Fatalf("Invalid license specifier: %v", err)
+	}
+
+	args := flag.Args()
+	if len(args) != 1 {
+		log.Fatal("module-dir must be passed as an argument.")
+	}
+	dir := args[0]
+
+	modules, err := fields.GetModules(dir)
+	if err != nil {
+		log.Fatalf("Error fetching modules: %v", err)
+	}
+
+	for _, module := range modules {
+		files, err := fields.CollectFiles(module, dir)
+		if err != nil {
+			log.Fatalf("Error fetching files for module %v: %v", module, err)
+		}
+
+		data, err := fields.GenerateFieldsYml(files)
+		if err != nil {
+			log.Fatalf("Error fetching files for module %v: %v", module, err)
+		}
+
+		encData, err := asset.EncodeData(string(data))
+		if err != nil {
+			log.Fatalf("Error encoding the data: %v", err)
+		}
+
+		var buf bytes.Buffer
+		asset.Template.Execute(&buf, asset.Data{
+			License: license,
+			Beat:    beatName,
+			Name:    module,
+			Data:    encData,
+			Package: module,
+		})
+
+		bs, err := format.Source(buf.Bytes())
+		if err != nil {
+			log.Fatalf("Error creating golang file from template: %v", err)
+		}
+
+		err = ioutil.WriteFile(path.Join(dir, module, "fields.go"), bs, 0644)
+		if err != nil {
+			log.Fatalf("Error writing fields.go: %v", err)
+		}
+	}
+}
+
+func usageFlag() {
+	fmt.Fprintf(os.Stderr, usageText)
+	flag.PrintDefaults()
+}

dev-tools/jenkins_ci.sh

@@ -31,6 +31,7 @@ cleanup() {
     echo "Cleaning stopped docker containers and dangling images/networks/volumes..."
     docker system prune -f || true
   fi
+
   echo "Cleanup complete."
 }
 trap cleanup EXIT