Port fields.yml collector to Golang

CHANGELOG-developer.asciidoc

@@ -21,6 +21,7 @@ The list below covers the major changes between 6.3.0 and master only.
 
 - The beat.Pipeline is now passed to cfgfile.RunnerFactory. Beats using libbeat for module reloading or autodiscovery need to be adapted. {pull}7018[7017]
 - Moving of TLS helper functions and structs from `output/tls` to `tlscommon`. {pull}7054[7054]
+- Port fields.yml collector to Golang {pull}6911[6911]
 
 ==== Bugfixes
 

auditbeat/Makefile

@@ -7,6 +7,7 @@ GOX_OS?=linux windows ## @Building List of all OS to be supported by "make cross
 DEV_OS?=linux
 TESTING_ENVIRONMENT?=snapshot-noxpack
 ES_BEATS?=..
+FIELDS_FILE_PATH=module
 
 # Path to the libbeat Makefile
 include ${ES_BEATS}/libbeat/scripts/Makefile
@@ -58,13 +59,6 @@ before-build:
 .PHONY: collect
 collect: fields collect-docs configs kibana
 
-# Collects all module and metricset fields
-.PHONY: fields
-fields: python-env
- @mkdir -p _meta
- @cp _meta/fields.common.yml _meta/fields.generated.yml
- @${PYTHON_ENV}/bin/python ${ES_BEATS}/metricbeat/scripts/fields_collector.py >> _meta/fields.generated.yml
-
 # Collects all module configs
 .PHONY: configs
 configs: python-env

filebeat/Makefile

@@ -5,6 +5,8 @@ SYSTEM_TESTS?=true
 TEST_ENVIRONMENT?=true
 GOX_FLAGS=-arch="amd64 386 arm ppc64 ppc64le"
 ES_BEATS?=..
+FIELDS_FILE_PATH=module
+
 
 include ${ES_BEATS}/libbeat/scripts/Makefile
 
@@ -19,13 +21,6 @@ kibana:
  @mkdir -p _meta/kibana
  @-cp -r module/*/_meta/kibana _meta/
 
-# Collects all module and dataset fields
-.PHONY: fields
-fields: python-env
- @mkdir -p _meta/
- @cp ${ES_BEATS}/filebeat/_meta/fields.common.yml _meta/fields.generated.yml
- @${PYTHON_ENV}/bin/python ${ES_BEATS}/metricbeat/scripts/fields_collector.py >> _meta/fields.generated.yml
-
 # Collects all modules files to be packaged in a temporary folder
 .PHONY: modules
 modules:

generator/beat/Makefile

@@ -1 +1,4 @@
+override FIELDS_FILE_PATH=
+export FIELDS_FILE_PATH
+
 include ../common/Makefile

generator/beat/{beat}/Makefile

@@ -46,4 +46,4 @@ before-build:
 
 # Collects all dependencies and then calls update
 .PHONY: collect
-collect:
+collect: fields

generator/common/Makefile

@@ -6,6 +6,7 @@ BEAT_PATH=${BUILD_DIR}/src/beatpath/testbeat
 ES_BEATS=${GOPATH}/src/github.com/elastic/beats
 PREPARE_COMMAND?=
 
+
 # Runs test build for mock beat
 .PHONY: test
 test: prepare-test

generator/metricbeat/Makefile

@@ -1,9 +1,16 @@
 BEAT_TYPE=metricbeat
-PREPARE_COMMAND=MODULE=elastic METRICSET=test make create-metricset ;
+PREPARE_COMMAND=MODULE=elastic METRICSET=test make create-metricset ; FIELDS_FILE_PATH=module
+override FIELDS_FILE_PATH=
+export FIELDS_FILE_PATH
+
 
 include ../common/Makefile
 
 prepare-test:: python-env
 
  mkdir -p ${BEAT_PATH}/scripts
  rsync -a --exclude=build ${PWD}/../../metricbeat/scripts/generate_imports_helper.py ${BEAT_PATH}/scripts
+
+# Collects all dependencies and then calls update
+.PHONY: collect
+collect: fields

heartbeat/Makefile

@@ -4,6 +4,7 @@ BEAT_PACKAGE_NAME=heartbeat-elastic
 BEAT_DESCRIPTION?=Ping remote services for availability and log results to Elasticsearch or send to Logstash.
 SYSTEM_TESTS=true
 TEST_ENVIRONMENT=false
+FIELDS_FILE_PATH=monitors/active
 
 # Path to the libbeat Makefile
 -include ../libbeat/scripts/Makefile
@@ -16,13 +17,6 @@ before-build:
 .PHONY: collect
 collect: fields imports
 
-# Collects all module and metricset fields
-.PHONY: fields
-fields:
- @mkdir -p _meta/
- @cp ${ES_BEATS}/heartbeat/_meta/fields.common.yml _meta/fields.generated.yml
- @cat ${ES_BEATS}/heartbeat/monitors/active/*/_meta/fields.yml >> _meta/fields.generated.yml
-
 # Generate imports for all monitors
 .PHONY: imports
 imports: python-env

libbeat/Makefile

@@ -1,15 +1,10 @@
 BEAT_NAME=libbeat
 TEST_ENVIRONMENT?=true
 SYSTEM_TESTS=true
+FIELDS_FILE_PATH=processors
 
 include scripts/Makefile
 
-# Collects all fields from processors
-.PHONY: fields
-fields:
- @cp _meta/fields.common.yml _meta/fields.generated.yml
- @cat processors/*/_meta/fields.yml >> _meta/fields.generated.yml
-
 # Collects all dependencies and then calls update
 .PHONY: collect
-collect: fields
+collect: libbeat_fields

libbeat/generator/fields/fields.go

@@ -0,0 +1,149 @@
+package fields
+
+import (
+ "bufio"
+ "fmt"
+ "io/ioutil"
+ "os"
+ "path"
+ "path/filepath"
+ "strings"
+)
+
+var (
+ generatedFieldsYml = filepath.Join("_meta", "fields.generated.yml")
+)
+
+// YmlFile holds the info on files and how to write them into the global fields.yml
+type YmlFile struct {
+ Path string
+ Indent int
+}
+
+func collectBeatFiles(beatPath string, fieldFiles []*YmlFile) ([]*YmlFile, error) {
+ commonFields := filepath.Join(beatPath, "_meta", "fields.common.yml")
+ _, err := os.Stat(commonFields)
+ if os.IsNotExist(err) {
+ return fieldFiles, nil
+ } else if err != nil {
+ return nil, err
+ }
+
+ files := []*YmlFile{
+ &YmlFile{
+ Path: commonFields,
+ Indent: 0,
+ },
+ }
+
+ return append(files, fieldFiles...), nil
+}
+
+func writeGeneratedFieldsYml(beatsPath string, fieldFiles []*YmlFile) error {
+ outPath := path.Join(beatsPath, generatedFieldsYml)
+ f, err := os.Create(outPath)
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+
+ fw := bufio.NewWriter(f)
+ for _, p := range fieldFiles {
+ ff, err := os.Open(p.Path)
+ if err != nil {
+ return err
+ }
+ defer ff.Close()
+
+ fs := bufio.NewScanner(ff)
+ for fs.Scan() {
+ err = writeIndentedLine(fw, fs.Text()+"\n", p.Indent)
+ if err != nil {
+ return err
+ }
+
+ }
+ if err := fs.Err(); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func writeIndentedLine(fw *bufio.Writer, l string, indent int) error {
+ ll := strings.Repeat(" ", indent) + l
+ fmt.Fprint(fw, ll)
+ return fw.Flush()
+}
+
+// Generate collects fields.yml files and concatenates them into one global file.
+func Generate(esBeatsPath, beatPath string, files []*YmlFile) error {
+ files, err := collectBeatFiles(beatPath, files)
+ if err != nil {
+ return err
+ }
+
+ err = writeGeneratedFieldsYml(beatPath, files)
+ if err != nil {
+ return err
+ }
+
+ return AppendFromLibbeat(esBeatsPath, beatPath)
+}
+
+// AppendFromLibbeat appends fields.yml of libbeat to the fields.yml
+func AppendFromLibbeat(esBeatsPath, beatPath string) error {
+ fieldsMetaPath := path.Join(beatPath, "_meta", "fields.yml")
+ generatedPath := path.Join(beatPath, generatedFieldsYml)
+
+ err := createIfNotExists(fieldsMetaPath, generatedPath)
+ if err != nil {
+ return err
+ }
+
+ if isLibbeat(beatPath) {
+ out := filepath.Join(esBeatsPath, "libbeat", "fields.yml")
+ return copyFileWithFlag(generatedPath, out, os.O_RDWR|os.O_CREATE|os.O_TRUNC)
+ }
+
+ libbeatPath := filepath.Join(esBeatsPath, "libbeat", generatedFieldsYml)
+ out := filepath.Join(beatPath, "fields.yml")
+ err = copyFileWithFlag(libbeatPath, out, os.O_RDWR|os.O_CREATE|os.O_TRUNC)
+ if err != nil {
+ return err
+ }
+ return copyFileWithFlag(generatedPath, out, os.O_WRONLY|os.O_APPEND)
+}
+
+func isLibbeat(beatPath string) bool {
+ return filepath.Base(beatPath) == "libbeat"
+}
+
+func createIfNotExists(inPath, outPath string) error {
+ _, err := os.Stat(outPath)
+ if os.IsNotExist(err) {
+ err := copyFileWithFlag(inPath, outPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC)
+ if err != nil {
+ fmt.Println("Cannot find _meta/fields.yml")
+ }
+ return nil
+ }
+ return err
+}
+
+func copyFileWithFlag(in, out string, flag int) error {
+ input, err := ioutil.ReadFile(in)
+ if err != nil {
+ return err
+ }
+
+ output, err := os.OpenFile(out, flag, 0644)
+ if err != nil {
+ return err
+ }
+ defer output.Close()
+
+ _, err = output.Write(input)
+ return err
+
+}

libbeat/generator/fields/module_fields_collector.go

@@ -0,0 +1,74 @@
+package fields
+
+import (
+ "io/ioutil"
+ "os"
+ "path/filepath"
+)
+
+var indentByModule = map[string]int{
+ "processors": 0,
+ "module": 8,
+ "active": 8,
+ "protos": 8,
+}
+
+// CollectModuleFiles looks for fields.yml files under the
+// specified root directory
+func CollectModuleFiles(root string) ([]*YmlFile, error) {
+ modules, err := ioutil.ReadDir(root)
+ if err != nil {
+ return nil, err
+ }
+
+ var files []*YmlFile
+ for _, m := range modules {
+ f, err := collectFiles(m, root)
+ if err != nil {
+ return nil, err
+ }
+ files = append(files, f...)
+ }
+
+ return files, nil
+}
+
+func collectFiles(module os.FileInfo, modulesPath string) ([]*YmlFile, error) {
+ if !module.IsDir() {
+ return nil, nil
+ }
+
+ var files []*YmlFile
+ fieldsYmlPath := filepath.Join(modulesPath, module.Name(), "_meta", "fields.yml")
+ if _, err := os.Stat(fieldsYmlPath); !os.IsNotExist(err) {
+ files = append(files, &YmlFile{
+ Path: fieldsYmlPath,
+ Indent: 0,
+ })
+ } else if !os.IsNotExist(err) && err != nil {
+ return nil, err
+ }
+
+ modulesRoot := filepath.Base(modulesPath)
+ sets, err := ioutil.ReadDir(filepath.Join(modulesPath, module.Name()))
+ if err != nil {
+ return nil, err
+ }
+
+ for _, s := range sets {
+ if !s.IsDir() {
+ continue
+ }
+
+ fieldsYmlPath = filepath.Join(modulesPath, module.Name(), s.Name(), "_meta", "fields.yml")
+ if _, err = os.Stat(fieldsYmlPath); !os.IsNotExist(err) {
+ files = append(files, &YmlFile{
+ Path: fieldsYmlPath,
+ Indent: indentByModule[modulesRoot],
+ })
+ } else if !os.IsNotExist(err) && err != nil {
+ return nil, err
+ }
+ }
+ return files, nil
+}

libbeat/scripts/Makefile

@@ -304,9 +304,19 @@ coverage-report:
  test ! -s ${COVERAGE_DIR}/unit.cov || go tool cover -html=${COVERAGE_DIR}/unit.cov -o ${COVERAGE_DIR}/unit.html
 
 
+.PHONY: fields
+fields:
+ echo $(PWD)
+ @go run ${ES_BEATS}/libbeat/scripts/cmd/global_fields/main.go --es_beats_path $(ES_BEATS) --beat_path $(PWD) $(FIELDS_FILE_PATH)
+
+
+.PHONY: libbeat_fields
+libbeat_fields:
+ @$(MAKE) -C ${ES_BEATS}/libbeat fields
+
 .PHONY: update
 update: ## @build Update expects the most recent version of libbeat in the GOPATH
-update: python-env collect
+update: python-env libbeat_fields collect
  @echo "Updating generated files for ${BEAT_NAME}"
 
  @# Update config files.
@@ -324,15 +334,7 @@ ifeq ($(BEAT_REF_YAML),true)
  @chmod 0640 ${BEAT_NAME}.reference.yml
 endif
 
- @# Update libbeat fields.generated.yml
- @$(MAKE) -C ${ES_BEATS}/libbeat fields
-
- @# Update fields.yml
- @test -s _meta/fields.generated.yml || cp _meta/fields.yml _meta/fields.generated.yml
-ifeq ($(BEAT_NAME), libbeat)
- @cp ${ES_BEATS}/libbeat/_meta/fields.generated.yml fields.yml
-else
- @cat ${ES_BEATS}/libbeat/_meta/fields.generated.yml _meta/fields.generated.yml > fields.yml
+ifneq ($(BEAT_NAME), libbeat)
  mkdir -p include
  go run ${ES_BEATS}/dev-tools/cmd/asset/asset.go -pkg include fields.yml $(BEAT_NAME) > include/fields.go
 endif