Simplify fields.yml generator (elastic#7713)

This PR simplifies the process of generating a new `fields.yml` file for a Beat.
* `libbeat/fields.yml` is not a dependency anymore
* only `fields.yml` is written to disk

Changes in the asset files are due to different number of newlines in generated `fields.yml` files.

### Interface changes of generator tools
Both `dev-tools/cmd/asset/asset.go` and `libbeat/scripts/cmd/global_fields/main.go` is changed. The first one has two new flags: `-in`, and `-out`. By default it reads from stdin and writes to stdout. Using the flags it's possible to read and write from other files. The second one has one new flag `-out` which specifies the output file. If it's not set, the output is written to stdout.

This makes it possible to generate asset files for Metricbeat without writing `libbeat/fields.yml` to disk. It will be useful in the future when more Beats have modular assets.

Closes elastic#7671

auditbeat/Makefile

@@ -12,7 +12,7 @@ include ${ES_BEATS}/libbeat/scripts/Makefile
 
 # Collects all dependencies and then calls update
 .PHONY: collect
-collect: fields collect-docs configs kibana
+collect: collect-docs configs kibana
 
 # Collects all module configs
 .PHONY: configs

dev-tools/cmd/asset/asset.go

@@ -20,6 +20,7 @@
 package main
 
 import (
+	"bufio"
 	"bytes"
 	"flag"
 	"fmt"
@@ -30,33 +31,54 @@ import (
 	"github.com/elastic/beats/libbeat/asset"
 )
 
-var pkg *string
+var (
+	pkg    string
+	input  string
+	output string
+)
 
 func init() {
-	pkg = flag.String("pkg", "", "Package name")
+	flag.StringVar(&pkg, "pkg", "", "Package name")
+	flag.StringVar(&input, "in", "-", "Source of input. \"-\" means reading from stdin")
+	flag.StringVar(&output, "out", "-", "Output path. \"-\" means writing to stdout")
 }
 
 func main() {
 	flag.Parse()
-
 	args := flag.Args()
-	if len(args) != 2 {
-		fmt.Fprintln(os.Stderr, "File path must be set")
-		os.Exit(1)
-	}
 
-	file := args[0]
-	beatName := args[1]
+	var (
+		file, beatName string
+		data           []byte
+		err            error
+	)
+	if input == "-" {
+		if len(args) != 2 {
+			fmt.Fprintln(os.Stderr, "File path must be set")
+			os.Exit(1)
+		}
+		file = args[0]
+		beatName = args[1]
 
-	data, err := ioutil.ReadFile(file)
-	if err != nil {
-		fmt.Fprintln(os.Stderr, "Invalid file path: %s", args[0])
-		os.Exit(1)
+		r := bufio.NewReader(os.Stdin)
+		data, err = ioutil.ReadAll(r)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Error while reading from stdin: %v\n", err)
+			os.Exit(1)
+		}
+	} else {
+		file = input
+		beatName = args[0]
+		data, err = ioutil.ReadFile(input)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Invalid file path: %s\n", input)
+			os.Exit(1)
+		}
 	}
 
 	encData, err := asset.EncodeData(string(data))
 	if err != nil {
-		fmt.Fprintln(os.Stderr, "Error encoding the data: %s", err)
+		fmt.Fprintf(os.Stderr, "Error encoding the data: %s\n", err)
 		os.Exit(1)
 	}
 
@@ -65,13 +87,17 @@ func main() {
 		Beat:    beatName,
 		Name:    file,
 		Data:    encData,
-		Package: *pkg,
+		Package: pkg,
 	})
 
 	bs, err := format.Source(buf.Bytes())
 	if err != nil {
 		panic(err)
 	}
 
-	os.Stdout.Write(bs)
+	if output == "-" {
+		os.Stdout.Write(bs)
+	} else {
+		ioutil.WriteFile(output, bs, 0640)
+	}
 }

dev-tools/jenkins_ci.ps1

@@ -28,11 +28,6 @@ echo "Fetching testing dependencies"
 exec { go get github.com/docker/libcompose }
 exec { go get github.com/jstemmer/go-junit-report }
 
-echo "Building libbeat fields.yml"
-cd libbeat
-exec { mage fields }
-cd ..
-
 if (Test-Path "$env:beat") {
     cd "$env:beat"
 } else {

dev-tools/mage/fields.go

@@ -42,6 +42,7 @@ func GenerateFieldsYAML(fieldsFiles ...string) error {
 		filepath.Join(beatsDir, globalFieldsCmdPath),
 		"-es_beats_path", beatsDir,
 		"-beat_path", CWD(),
+		"-out", "fields.yml",
 	)
 
 	return globalFieldsCmd(fieldsFiles...)

filebeat/Makefile

@@ -42,7 +42,7 @@ imports: python-env
 
 # Runs all collection steps and updates afterwards
 .PHONY: collect
-collect: fields kibana configs collect-docs imports
+collect: kibana configs collect-docs imports
 
 # Creates a new module. Requires the params MODULE
 .PHONY: create-module

generator/beat/{beat}/Makefile

@@ -35,4 +35,4 @@ git-add:
 
 # Collects all dependencies and then calls update
 .PHONY: collect
-collect: fields
+collect:

generator/metricbeat/Makefile

@@ -10,5 +10,4 @@ prepare-test:: python-env
 
 # Collects all dependencies and then calls update
 .PHONY: collect
-collect: fields
-
+collect:

heartbeat/Makefile

@@ -8,7 +8,7 @@ TEST_ENVIRONMENT=false
 
 # Collects all dependencies and then calls update
 .PHONY: collect
-collect: fields imports kibana
+collect: imports kibana
 
 # Generate imports for all monitors
 .PHONY: imports

libbeat/Makefile

@@ -6,4 +6,4 @@ include scripts/Makefile
 
 # Collects all dependencies and then calls update
 .PHONY: collect
-collect: libbeat_fields
+collect:

libbeat/generator/fields/fields.go

@@ -20,17 +20,9 @@ package fields
 import (
 	"bufio"
 	"bytes"
-	"io/ioutil"
 	"os"
-	"path"
 	"path/filepath"
 	"strings"
-
-	"github.com/pkg/errors"
-)
-
-var (
-	generatedFieldsYml = filepath.Join("_meta", "fields.generated.yml")
 )
 
 // YmlFile holds the info on files and how to write them into the global fields.yml
@@ -39,37 +31,67 @@ type YmlFile struct {
 	Indent int
 }
 
-func collectBeatFiles(beatPath string, fieldFiles []*YmlFile) ([]*YmlFile, error) {
-	commonFields := filepath.Join(beatPath, "_meta", "fields.common.yml")
-	_, err := os.Stat(commonFields)
-	if os.IsNotExist(err) {
-		return fieldFiles, nil
-	} else if err != nil {
-		return nil, err
+func collectCommonFiles(esBeatsPath, beatPath string, fieldFiles []*YmlFile) ([]*YmlFile, error) {
+	commonFields := []string{
+		filepath.Join(beatPath, "_meta/fields.common.yml"),
+	}
+
+	var libbeatFieldFiles []*YmlFile
+	var err error
+	if !isLibbeat(beatPath) {
+		commonFields = append(commonFields,
+			filepath.Join(esBeatsPath, "libbeat/_meta/fields.common.yml"),
+		)
+
+		libbeatModulesPath := filepath.Join(esBeatsPath, "libbeat/processors")
+		libbeatFieldFiles, err = CollectModuleFiles(libbeatModulesPath)
+		if err != nil {
+			return nil, err
+		}
 	}
 
-	files := []*YmlFile{
-		{
-			Path:   commonFields,
+	var files []*YmlFile
+	for _, cf := range commonFields {
+		_, err := os.Stat(cf)
+		if os.IsNotExist(err) {
+			continue
+		} else if err != nil {
+			return nil, err
+		}
+		files = append(files, &YmlFile{
+			Path:   cf,
 			Indent: 0,
-		},
+		})
 	}
 
+	files = append(files, libbeatFieldFiles...)
+
 	return append(files, fieldFiles...), nil
 }
 
-func writeGeneratedFieldsYml(beatsPath string, fieldFiles []*YmlFile) error {
-	outPath := path.Join(beatsPath, generatedFieldsYml)
-	f, err := os.Create(outPath)
+func isLibbeat(beatPath string) bool {
+	return filepath.Base(beatPath) == "libbeat"
+}
+
+func writeGeneratedFieldsYml(beatPath string, fieldFiles []*YmlFile, output string) error {
+	data, err := GenerateFieldsYml(fieldFiles)
 	if err != nil {
 		return err
 	}
-	defer f.Close()
 
-	data, err := GenerateFieldsYml(fieldFiles)
+	if output == "-" {
+		fw := bufio.NewWriter(os.Stdout)
+		fw.Write(data)
+		return fw.Flush()
+	}
+
+	outPath := filepath.Join(beatPath, output)
+	f, err := os.Create(outPath)
 	if err != nil {
 		return err
 	}
+	defer f.Close()
+
 	fw := bufio.NewWriter(f)
 	fw.Write(data)
 	return fw.Flush()
@@ -106,77 +128,11 @@ func writeIndentedLine(buf *bytes.Buffer, line string, indent int) error {
 }
 
 // Generate collects fields.yml files and concatenates them into one global file.
-func Generate(esBeatsPath, beatPath string, files []*YmlFile) error {
-	files, err := collectBeatFiles(beatPath, files)
-	if err != nil {
-		return err
-	}
-
-	err = writeGeneratedFieldsYml(beatPath, files)
-	if err != nil {
-		return err
-	}
-
-	return AppendFromLibbeat(esBeatsPath, beatPath)
-}
-
-// AppendFromLibbeat appends fields.yml of libbeat to the fields.yml
-func AppendFromLibbeat(esBeatsPath, beatPath string) error {
-	fieldsMetaPath := path.Join(beatPath, "_meta", "fields.yml")
-	generatedPath := path.Join(beatPath, generatedFieldsYml)
-
-	err := createIfNotExists(fieldsMetaPath, generatedPath)
-	if err != nil {
-		return err
-	}
-
-	if isLibbeat(beatPath) {
-		out := filepath.Join(esBeatsPath, "libbeat", "fields.yml")
-		return copyFileWithFlag(generatedPath, out, os.O_RDWR|os.O_CREATE|os.O_TRUNC)
-	}
-
-	libbeatPath := filepath.Join(esBeatsPath, "libbeat", generatedFieldsYml)
-	out := filepath.Join(beatPath, "fields.yml")
-	err = copyFileWithFlag(libbeatPath, out, os.O_RDWR|os.O_CREATE|os.O_TRUNC)
+func Generate(esBeatsPath, beatPath string, files []*YmlFile, output string) error {
+	files, err := collectCommonFiles(esBeatsPath, beatPath, files)
 	if err != nil {
 		return err
 	}
-	return copyFileWithFlag(generatedPath, out, os.O_WRONLY|os.O_APPEND)
-}
-
-func isLibbeat(beatPath string) bool {
-	return filepath.Base(beatPath) == "libbeat"
-}
 
-func createIfNotExists(inPath, outPath string) error {
-	_, err := os.Stat(outPath)
-	if os.IsNotExist(err) {
-		err := copyFileWithFlag(inPath, outPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC)
-		if err != nil {
-			return err
-		}
-		return nil
-	}
-	return err
-}
-
-func copyFileWithFlag(in, out string, flag int) error {
-	input, err := ioutil.ReadFile(in)
-	if err != nil {
-		return errors.Wrap(err, "failed to read source in copy")
-	}
-
-	if err := os.MkdirAll(filepath.Dir(out), 0755); err != nil {
-		return errors.Wrapf(err, "failed to create destination dir for copy "+
-			"at %v", filepath.Dir(out))
-	}
-
-	output, err := os.OpenFile(out, flag, 0644)
-	if err != nil {
-		return errors.Wrap(err, "failed to open destination file for copy")
-	}
-	defer output.Close()
-
-	_, err = output.Write(input)
-	return err
+	return writeGeneratedFieldsYml(beatPath, files, output)
 }

libbeat/generator/fields/module_fields_collector.go

@@ -71,7 +71,7 @@ func CollectModuleFiles(modulesDir string) ([]*YmlFile, error) {
 func CollectFiles(module string, modulesPath string) ([]*YmlFile, error) {
 
 	var files []*YmlFile
-	fieldsYmlPath := filepath.Join(modulesPath, module, "_meta", "fields.yml")
+	fieldsYmlPath := filepath.Join(modulesPath, module, "_meta/fields.yml")
 	if _, err := os.Stat(fieldsYmlPath); !os.IsNotExist(err) {
 		files = append(files, &YmlFile{
 			Path:   fieldsYmlPath,
@@ -91,7 +91,7 @@ func CollectFiles(module string, modulesPath string) ([]*YmlFile, error) {
 		if !s.IsDir() {
 			continue
 		}
-		fieldsYmlPath = filepath.Join(modulesPath, module, s.Name(), "_meta", "fields.yml")
+		fieldsYmlPath = filepath.Join(modulesPath, module, s.Name(), "_meta/fields.yml")
 		if _, err = os.Stat(fieldsYmlPath); !os.IsNotExist(err) {
 			files = append(files, &YmlFile{
 				Path:   fieldsYmlPath,

libbeat/kibana/fields_transformer.go

@@ -80,7 +80,7 @@ func (t *fieldsTransformer) transformFields(commonFields common.Fields, path str
 		}
 
 		if t.keys[f.Path] != nil {
-			msg := fmt.Sprintf("ERROR: Field <%s> is duplicated. Please update and try again.", f.Path)
+			msg := fmt.Sprintf("ERROR: Field <%s> is duplicated. Please update and try again.\n", f.Path)
 			panic(errors.New(msg))
 		}