Simplify fields.yml generator (elastic#7713)

This PR simplifies the process of generating a new `fields.yml` file for a Beat.
* `libbeat/fields.yml` is not a dependency anymore
* only `fields.yml` is written to disk

Changes in the asset files are due to different number of newlines in generated `fields.yml` files.

### Interface changes of generator tools
Both `dev-tools/cmd/asset/asset.go` and `libbeat/scripts/cmd/global_fields/main.go` is changed. The first one has two new flags: `-in`, and `-out`. By default it reads from stdin and writes to stdout. Using the flags it's possible to read and write from other files. The second one has one new flag `-out` which specifies the output file. If it's not set, the output is written to stdout.

This makes it possible to generate asset files for Metricbeat without writing `libbeat/fields.yml` to disk. It will be useful in the future when more Beats have modular assets.

Closes elastic#7671
(cherry picked from commit 5797005)

auditbeat/Makefile

@@ -12,7 +12,7 @@ include ${ES_BEATS}/libbeat/scripts/Makefile
 
 # Collects all dependencies and then calls update
 .PHONY: collect
-collect: fields collect-docs configs kibana
+collect: collect-docs configs kibana
 
 # Collects all module configs
 .PHONY: configs

dev-tools/cmd/asset/asset.go

@@ -20,6 +20,7 @@
 package main
 
 import (
+ "bufio"
  "bytes"
  "flag"
  "fmt"
@@ -30,33 +31,54 @@ import (
  "github.com/elastic/beats/libbeat/asset"
 )
 
-var pkg *string
+var (
+ pkg string
+ input string
+ output string
+)
 
 func init() {
- pkg = flag.String("pkg", "", "Package name")
+ flag.StringVar(&pkg, "pkg", "", "Package name")
+ flag.StringVar(&input, "in", "-", "Source of input. \"-\" means reading from stdin")
+ flag.StringVar(&output, "out", "-", "Output path. \"-\" means writing to stdout")
 }
 
 func main() {
  flag.Parse()
-
  args := flag.Args()
- if len(args) != 2 {
- fmt.Fprintln(os.Stderr, "File path must be set")
- os.Exit(1)
- }
 
- file := args[0]
- beatName := args[1]
+ var (
+ file, beatName string
+ data []byte
+ err error
+ )
+ if input == "-" {
+ if len(args) != 2 {
+ fmt.Fprintln(os.Stderr, "File path must be set")
+ os.Exit(1)
+ }
+ file = args[0]
+ beatName = args[1]
 
- data, err := ioutil.ReadFile(file)
- if err != nil {
- fmt.Fprintln(os.Stderr, "Invalid file path: %s", args[0])
- os.Exit(1)
+ r := bufio.NewReader(os.Stdin)
+ data, err = ioutil.ReadAll(r)
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Error while reading from stdin: %v\n", err)
+ os.Exit(1)
+ }
+ } else {
+ file = input
+ beatName = args[0]
+ data, err = ioutil.ReadFile(input)
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Invalid file path: %s\n", input)
+ os.Exit(1)
+ }
  }
 
  encData, err := asset.EncodeData(string(data))
  if err != nil {
- fmt.Fprintln(os.Stderr, "Error encoding the data: %s", err)
+ fmt.Fprintf(os.Stderr, "Error encoding the data: %s\n", err)
  os.Exit(1)
  }
 
@@ -65,13 +87,17 @@ func main() {
  Beat: beatName,
  Name: file,
  Data: encData,
- Package: *pkg,
+ Package: pkg,
  })
 
  bs, err := format.Source(buf.Bytes())
  if err != nil {
  panic(err)
  }
 
- os.Stdout.Write(bs)
+ if output == "-" {
+ os.Stdout.Write(bs)
+ } else {
+ ioutil.WriteFile(output, bs, 0640)
+ }
 }

dev-tools/jenkins_ci.ps1

@@ -24,11 +24,6 @@ $env:MAGEFILE_CACHE = "$env:WORKSPACE\.magefile"
 exec { go install github.com/elastic/beats/vendor/github.com/magefile/mage }
 exec { go get -u github.com/jstemmer/go-junit-report }
 
-echo "Building libbeat fields.yml"
-cd libbeat
-exec { mage fields }
-cd ..
-
 if (Test-Path "$env:beat") {
  cd "$env:beat"
 } else {

dev-tools/mage/fields.go

@@ -42,6 +42,7 @@ func GenerateFieldsYAML(fieldsFiles ...string) error {
  filepath.Join(beatsDir, globalFieldsCmdPath),
  "-es_beats_path", beatsDir,
  "-beat_path", CWD(),
+ "-out", "fields.yml",
  )
 
  return globalFieldsCmd(fieldsFiles...)

filebeat/Makefile

@@ -42,7 +42,7 @@ imports: python-env
 
 # Runs all collection steps and updates afterwards
 .PHONY: collect
-collect: fields kibana configs collect-docs imports
+collect: kibana configs collect-docs imports
 
 # Creates a new module. Requires the params MODULE
 .PHONY: create-module

generator/beat/{beat}/Makefile

@@ -35,4 +35,4 @@ git-add:
 
 # Collects all dependencies and then calls update
 .PHONY: collect
-collect: fields
+collect:

generator/metricbeat/Makefile

@@ -10,5 +10,4 @@ prepare-test:: python-env
 
 # Collects all dependencies and then calls update
 .PHONY: collect
-collect: fields
-
+collect:

heartbeat/Makefile

@@ -8,7 +8,7 @@ TEST_ENVIRONMENT=false
 
 # Collects all dependencies and then calls update
 .PHONY: collect
-collect: fields imports kibana
+collect: imports kibana
 
 # Generate imports for all monitors
 .PHONY: imports

libbeat/Makefile

@@ -6,4 +6,4 @@ include scripts/Makefile
 
 # Collects all dependencies and then calls update
 .PHONY: collect
-collect: libbeat_fields
+collect:

libbeat/generator/fields/fields.go

@@ -20,17 +20,9 @@ package fields
 import (
  "bufio"
  "bytes"
- "io/ioutil"
  "os"
- "path"
  "path/filepath"
  "strings"
-
- "github.com/pkg/errors"
-)
-
-var (
- generatedFieldsYml = filepath.Join("_meta", "fields.generated.yml")
 )
 
 // YmlFile holds the info on files and how to write them into the global fields.yml
@@ -39,37 +31,67 @@ type YmlFile struct {
  Indent int
 }
 
-func collectBeatFiles(beatPath string, fieldFiles []*YmlFile) ([]*YmlFile, error) {
- commonFields := filepath.Join(beatPath, "_meta", "fields.common.yml")
- _, err := os.Stat(commonFields)
- if os.IsNotExist(err) {
- return fieldFiles, nil
- } else if err != nil {
- return nil, err
+func collectCommonFiles(esBeatsPath, beatPath string, fieldFiles []*YmlFile) ([]*YmlFile, error) {
+ commonFields := []string{
+ filepath.Join(beatPath, "_meta/fields.common.yml"),
+ }
+
+ var libbeatFieldFiles []*YmlFile
+ var err error
+ if !isLibbeat(beatPath) {
+ commonFields = append(commonFields,
+ filepath.Join(esBeatsPath, "libbeat/_meta/fields.common.yml"),
+ )
+
+ libbeatModulesPath := filepath.Join(esBeatsPath, "libbeat/processors")
+ libbeatFieldFiles, err = CollectModuleFiles(libbeatModulesPath)
+ if err != nil {
+ return nil, err
+ }
  }
 
- files := []*YmlFile{
- {
- Path: commonFields,
+ var files []*YmlFile
+ for _, cf := range commonFields {
+ _, err := os.Stat(cf)
+ if os.IsNotExist(err) {
+ continue
+ } else if err != nil {
+ return nil, err
+ }
+ files = append(files, &YmlFile{
+ Path: cf,
  Indent: 0,
- },
+ })
  }
 
+ files = append(files, libbeatFieldFiles...)
+
  return append(files, fieldFiles...), nil
 }
 
-func writeGeneratedFieldsYml(beatsPath string, fieldFiles []*YmlFile) error {
- outPath := path.Join(beatsPath, generatedFieldsYml)
- f, err := os.Create(outPath)
+func isLibbeat(beatPath string) bool {
+ return filepath.Base(beatPath) == "libbeat"
+}
+
+func writeGeneratedFieldsYml(beatPath string, fieldFiles []*YmlFile, output string) error {
+ data, err := GenerateFieldsYml(fieldFiles)
  if err != nil {
  return err
  }
- defer f.Close()
 
- data, err := GenerateFieldsYml(fieldFiles)
+ if output == "-" {
+ fw := bufio.NewWriter(os.Stdout)
+ fw.Write(data)
+ return fw.Flush()
+ }
+
+ outPath := filepath.Join(beatPath, output)
+ f, err := os.Create(outPath)
  if err != nil {
  return err
  }
+ defer f.Close()
+
  fw := bufio.NewWriter(f)
  fw.Write(data)
  return fw.Flush()
@@ -106,77 +128,11 @@ func writeIndentedLine(buf *bytes.Buffer, line string, indent int) error {
 }
 
 // Generate collects fields.yml files and concatenates them into one global file.
-func Generate(esBeatsPath, beatPath string, files []*YmlFile) error {
- files, err := collectBeatFiles(beatPath, files)
- if err != nil {
- return err
- }
-
- err = writeGeneratedFieldsYml(beatPath, files)
- if err != nil {
- return err
- }
-
- return AppendFromLibbeat(esBeatsPath, beatPath)
-}
-
-// AppendFromLibbeat appends fields.yml of libbeat to the fields.yml
-func AppendFromLibbeat(esBeatsPath, beatPath string) error {
- fieldsMetaPath := path.Join(beatPath, "_meta", "fields.yml")
- generatedPath := path.Join(beatPath, generatedFieldsYml)
-
- err := createIfNotExists(fieldsMetaPath, generatedPath)
- if err != nil {
- return err
- }
-
- if isLibbeat(beatPath) {
- out := filepath.Join(esBeatsPath, "libbeat", "fields.yml")
- return copyFileWithFlag(generatedPath, out, os.O_RDWR|os.O_CREATE|os.O_TRUNC)
- }
-
- libbeatPath := filepath.Join(esBeatsPath, "libbeat", generatedFieldsYml)
- out := filepath.Join(beatPath, "fields.yml")
- err = copyFileWithFlag(libbeatPath, out, os.O_RDWR|os.O_CREATE|os.O_TRUNC)
+func Generate(esBeatsPath, beatPath string, files []*YmlFile, output string) error {
+ files, err := collectCommonFiles(esBeatsPath, beatPath, files)
  if err != nil {
  return err
  }
- return copyFileWithFlag(generatedPath, out, os.O_WRONLY|os.O_APPEND)
-}
-
-func isLibbeat(beatPath string) bool {
- return filepath.Base(beatPath) == "libbeat"
-}
 
-func createIfNotExists(inPath, outPath string) error {
- _, err := os.Stat(outPath)
- if os.IsNotExist(err) {
- err := copyFileWithFlag(inPath, outPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC)
- if err != nil {
- return err
- }
- return nil
- }
- return err
-}
-
-func copyFileWithFlag(in, out string, flag int) error {
- input, err := ioutil.ReadFile(in)
- if err != nil {
- return errors.Wrap(err, "failed to read source in copy")
- }
-
- if err := os.MkdirAll(filepath.Dir(out), 0755); err != nil {
- return errors.Wrapf(err, "failed to create destination dir for copy "+
- "at %v", filepath.Dir(out))
- }
-
- output, err := os.OpenFile(out, flag, 0644)
- if err != nil {
- return errors.Wrap(err, "failed to open destination file for copy")
- }
- defer output.Close()
-
- _, err = output.Write(input)
- return err
+ return writeGeneratedFieldsYml(beatPath, files, output)
 }

libbeat/generator/fields/module_fields_collector.go

@@ -71,7 +71,7 @@ func CollectModuleFiles(modulesDir string) ([]*YmlFile, error) {
 func CollectFiles(module string, modulesPath string) ([]*YmlFile, error) {
 
  var files []*YmlFile
- fieldsYmlPath := filepath.Join(modulesPath, module, "_meta", "fields.yml")
+ fieldsYmlPath := filepath.Join(modulesPath, module, "_meta/fields.yml")
  if _, err := os.Stat(fieldsYmlPath); !os.IsNotExist(err) {
  files = append(files, &YmlFile{
  Path: fieldsYmlPath,
@@ -91,7 +91,7 @@ func CollectFiles(module string, modulesPath string) ([]*YmlFile, error) {
  if !s.IsDir() {
  continue
  }
- fieldsYmlPath = filepath.Join(modulesPath, module, s.Name(), "_meta", "fields.yml")
+ fieldsYmlPath = filepath.Join(modulesPath, module, s.Name(), "_meta/fields.yml")
  if _, err = os.Stat(fieldsYmlPath); !os.IsNotExist(err) {
  files = append(files, &YmlFile{
  Path: fieldsYmlPath,

libbeat/kibana/fields_transformer.go

@@ -80,7 +80,7 @@ func (t *fieldsTransformer) transformFields(commonFields common.Fields, path str
  }
 
  if t.keys[f.Path] != nil {
- msg := fmt.Sprintf("ERROR: Field <%s> is duplicated. Please update and try again.", f.Path)
+ msg := fmt.Sprintf("ERROR: Field <%s> is duplicated. Please update and try again.\n", f.Path)
  panic(errors.New(msg))
  }