Cherry-pick #8693 to 6.x: Add Suricata module to Filebeat

CHANGELOG-developer.asciidoc

@@ -29,6 +29,7 @@ The list below covers the major changes between 6.3.0 and master only.
  the building a Beat more cross-platform friendly (e.g. Windows). This requires that your Beat
  has a magefile.go with a fields target. The `FIELDS_FILE_PATH` make variable is no longer
  used because the value is specified in magefile.go. {pull}7670[7670]
+- Renamed `-beat-name` CLI option used in `kibana_index_pattern.go` to `-beat` for consistency with other scripts in `dev-tools/cmd`. {pull}8615[8615]
 
 ==== Bugfixes
 
@@ -52,3 +53,9 @@ The list below covers the major changes between 6.3.0 and master only.
 - Set current year in generator templates. {pull}8396[8396]
 - You can now override default settings of libbeat by using instance.Settings. {pull}8449[8449]
 - Add `-space-id` option to `export_dashboards.go` script to support Kibana Spaces {pull}7942[7942]
+- Add `-name` option to `asset.go` script to explicitly name the asset rather than using its filename. {pull}8693[8693]
+- Add `-out` option to `kibana_index_pattern.go` to control the output dir to make it possible to write the generated output to `build/kibana` instead of `_meta/kibana.generated` (but the output dir remains unchanged at this point). {pull}8615[8615]
+- Add `module_fields.go` for generated `fields.go` files for modules. {pull}8615[8615]
+- Add `mage.GenerateModuleReferenceConfig` for generating reference config files that include configuration sections from the module directory. {pull}8615[8615]
+- Add `mage.GenerateFieldsGo` for generating fields.go files. {pull}8615[8615]
+- Add `mage.KibanaDashboards` for collecting Kibana dashboards and generating index patterns. {pull}8615[8615]

CHANGELOG.asciidoc

@@ -122,6 +122,7 @@ https://github.com/elastic/beats/compare/v6.4.0...6.x[Check the HEAD diff]
 - Release `docker` input as GA. {pull}8328[8328]
 - Keep unparsed user agent information in user_agent.original. {pull}7823[7832]
 - Added default and TCP parsing formats to HAproxy module {issue}8311[8311] {pull}8637[8637]
+- Add Suricata IDS/IDP/NSM module. {issue}8153[8153] {pull}8693[8693]
 
 *Heartbeat*
 

dev-tools/cmd/asset/asset.go

@@ -37,6 +37,7 @@ var (
  pkg string
  input string
  output string
+ name string
  license = "ASL2"
 )
 
@@ -45,6 +46,7 @@ func init() {
  flag.StringVar(&input, "in", "-", "Source of input. \"-\" means reading from stdin")
  flag.StringVar(&output, "out", "-", "Output path. \"-\" means writing to stdout")
  flag.StringVar(&license, "license", "ASL2", "License header for generated file.")
+ flag.StringVar(&name, "name", "", "Asset name")
 }
 
 func main() {
@@ -94,9 +96,12 @@ func main() {
  os.Exit(1)
  }
  var buf bytes.Buffer
+ if name == "" {
+ name = file
+ }
  asset.Template.Execute(&buf, asset.Data{
  Beat: beatName,
- Name: file,
+ Name: name,
  Data: encData,
  License: licenseHeader,
  Package: pkg,

dev-tools/cmd/kibana_index_pattern/kibana_index_pattern.go

@@ -19,51 +19,73 @@ package main
 
 import (
  "flag"
- "fmt"
- "os"
+ "log"
+ "path/filepath"
 
  "github.com/elastic/beats/libbeat/common"
  "github.com/elastic/beats/libbeat/kibana"
  "github.com/elastic/beats/libbeat/version"
 )
 
+var usageText = `
+Usage: kibana_index_pattern [flags]
+ kibana_index_pattern generates Kibana index patterns from the Beat's
+ fields.yml file. It will create a index pattern file that is usable with both
+ Kibana 5.x and 6.x.
+Options:
+`[1:]
+
+var (
+ beatName string
+ beatVersion string
+ indexPattern string
+ fieldsYAMLFile string
+ outputDir string
+)
+
+func init() {
+ flag.StringVar(&beatName, "beat", "", "Name of the beat. (Required)")
+ flag.StringVar(&beatVersion, "version", version.GetDefaultVersion(), "Beat version. (Required)")
+ flag.StringVar(&indexPattern, "index", "", "Kibana index pattern. (Required)")
+ flag.StringVar(&fieldsYAMLFile, "fields", "fields.yml", "fields.yml file containing all fields used by the Beat.")
+ flag.StringVar(&outputDir, "out", "build/kibana", "Output dir.")
+}
+
 func main() {
- index := flag.String("index", "", "The name of the index pattern. (required)")
- beatName := flag.String("beat-name", "", "The name of the beat. (required)")
- beatDir := flag.String("beat-dir", "", "The local beat directory. (required)")
- beatVersion := flag.String("version", version.GetDefaultVersion(), "The beat version.")
+ log.SetFlags(0)
  flag.Parse()
 
- if *index == "" {
- fmt.Fprint(os.Stderr, "The name of the index pattern must be set.")
- os.Exit(1)
+ if beatName == "" {
+ log.Fatal("Name of the Beat must be set (-beat).")
  }
 
- if *beatName == "" {
- fmt.Fprint(os.Stderr, "The name of the beat must be set.")
- os.Exit(1)
+ if beatVersion == "" {
+ log.Fatal("Beat version must be set (-version).")
  }
 
- if *beatDir == "" {
- fmt.Fprint(os.Stderr, "The beat directory must be set.")
- os.Exit(1)
+ if indexPattern == "" {
+ log.Fatal("Index pattern must be set (-index).")
  }
 
  version5, _ := common.NewVersion("5.0.0")
  version6, _ := common.NewVersion("6.0.0")
- versions := []*common.Version{version5, version6}
+ versions := []common.Version{*version5, *version6}
  for _, version := range versions {
+ indexPattern, err := kibana.NewGenerator(indexPattern, beatName, fieldsYAMLFile, outputDir, beatVersion, version)
+ if err != nil {
+ log.Fatal(err)
+ }
 
- indexPatternGenerator, err := kibana.NewGenerator(*index, *beatName, *beatDir, *beatVersion, *version)
+ file, err := indexPattern.Generate()
  if err != nil {
- fmt.Fprintf(os.Stderr, err.Error())
- os.Exit(1)
+ log.Fatal(err)
  }
- pattern, err := indexPatternGenerator.Generate()
+
+ // Log output file location.
+ absFile, err := filepath.Abs(file)
  if err != nil {
- fmt.Fprintf(os.Stderr, err.Error())
- os.Exit(1)
+ absFile = file
  }
- fmt.Fprintf(os.Stdout, "-- The index pattern was created under %v\n", pattern)
+ log.Printf(">> The index pattern was created under %v", absFile)
  }
 }

dev-tools/cmd/module_fields/main.go

@@ -0,0 +1,118 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package main
+
+import (
+ "bytes"
+ "flag"
+ "fmt"
+ "go/format"
+ "io/ioutil"
+ "log"
+ "os"
+ "path"
+
+ "github.com/elastic/beats/libbeat/asset"
+ "github.com/elastic/beats/libbeat/generator/fields"
+ "github.com/elastic/beats/licenses"
+)
+
+var usageText = `
+Usage: module_fields [flags] [module-dir]
+ module_fields generates a fields.go file containing a copy of the module's
+ field.yml data in a format that can be embedded in Beat's binary. module-dir
+ should be the directory containing modules (e.g. filebeat/module).
+Options:
+`[1:]
+
+var (
+ beatName string
+ license string
+)
+
+func init() {
+ flag.StringVar(&beatName, "beat", "", "Name of the beat. (Required)")
+ flag.StringVar(&license, "license", "ASL2", "License header for generated file.")
+ flag.Usage = usageFlag
+}
+
+func main() {
+ log.SetFlags(0)
+ flag.Parse()
+
+ if beatName == "" {
+ log.Fatal("You must use -beat to specify the beat name.")
+ }
+
+ license, err := licenses.Find(license)
+ if err != nil {
+ log.Fatalf("Invalid license specifier: %v", err)
+ }
+
+ args := flag.Args()
+ if len(args) != 1 {
+ log.Fatal("module-dir must be passed as an argument.")
+ }
+ dir := args[0]
+
+ modules, err := fields.GetModules(dir)
+ if err != nil {
+ log.Fatalf("Error fetching modules: %v", err)
+ }
+
+ for _, module := range modules {
+ files, err := fields.CollectFiles(module, dir)
+ if err != nil {
+ log.Fatalf("Error fetching files for module %v: %v", module, err)
+ }
+
+ data, err := fields.GenerateFieldsYml(files)
+ if err != nil {
+ log.Fatalf("Error fetching files for module %v: %v", module, err)
+ }
+
+ encData, err := asset.EncodeData(string(data))
+ if err != nil {
+ log.Fatalf("Error encoding the data: %v", err)
+ }
+
+ var buf bytes.Buffer
+ asset.Template.Execute(&buf, asset.Data{
+ License: license,
+ Beat: beatName,
+ Name: module,
+ Data: encData,
+ Package: module,
+ })
+
+ bs, err := format.Source(buf.Bytes())
+ if err != nil {
+ log.Fatalf("Error creating golang file from template: %v", err)
+ }
+
+ err = ioutil.WriteFile(path.Join(dir, module, "fields.go"), bs, 0644)
+ if err != nil {
+ log.Fatalf("Error writing fields.go: %v", err)
+ }
+ }
+}
+
+func usageFlag() {
+ fmt.Fprintf(os.Stderr, usageText)
+ flag.PrintDefaults()
+}

dev-tools/jenkins_ci.sh

@@ -31,6 +31,7 @@ cleanup() {
  echo "Cleaning stopped docker containers and dangling images/networks/volumes..."
  docker system prune -f || true
  fi
+
  echo "Cleanup complete."
 }
 trap cleanup EXIT