From 49056405865ae85e2f5fb9be5786d3afbc83614d Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Fri, 20 Jul 2018 00:38:14 -0400 Subject: [PATCH 1/2] Generate fields.yml using Mage Make will now delegate to mage for generating fields.yml. Make will check if the mage command exists and go install it if not. The FIELDS_FILE_PATH make variable is not longer used because the path(s) are specified in magefile.go. This allows fields.yml to be generated on Windows using Mage. The CI scripts for Windows have been updated so that fields.yml is generated for all Beats during testing. This also adds a make.bat in each directory where building occurs to give Windows users a starting point. Some fixes were made to the generators because: - rsync was excluding important source files contained in a directory named "build" - the generated project needed to be `git init` before running certain magefile targets that detect project's root dir and import path. --- CHANGELOG-developer.asciidoc | 4 ++ auditbeat/Makefile | 1 - auditbeat/magefile.go | 5 ++ auditbeat/make.bat | 11 +++++ dev-tools/jenkins_ci.ps1 | 23 +++++---- dev-tools/mage/fields.go | 48 +++++++++++++++++++ .../templates/common/magefile.go.tmpl | 5 ++ filebeat/magefile.go | 5 ++ filebeat/make.bat | 18 ++++--- generator/beat/Makefile | 3 -- generator/beat/{beat}/Makefile | 21 ++++---- generator/beat/{beat}/magefile.go | 5 ++ generator/beat/{beat}/make.bat | 11 +++++ generator/common/Makefile | 19 ++++++-- generator/metricbeat/Makefile | 10 ++-- generator/metricbeat/{beat}/Makefile | 9 ++-- generator/metricbeat/{beat}/magefile.go | 5 ++ generator/metricbeat/{beat}/make.bat | 11 +++++ heartbeat/Makefile | 1 - heartbeat/magefile.go | 5 ++ heartbeat/make.bat | 11 +++++ libbeat/Makefile | 1 - libbeat/generator/fields/fields.go | 14 ++++-- libbeat/magefile.go | 39 +++++++++++++++ libbeat/make.bat | 11 +++++ libbeat/scripts/Makefile | 11 +++-- libbeat/scripts/cmd/global_fields/main.go | 37 +++++++------- make.bat | 11 +++++ metricbeat/Makefile | 1 - metricbeat/magefile.go | 5 ++ metricbeat/make.bat | 35 +++----------- packetbeat/Makefile | 1 - packetbeat/magefile.go | 5 ++ packetbeat/make.bat | 11 +++++ winlogbeat/magefile.go | 5 ++ winlogbeat/make.bat | 35 +++----------- 36 files changed, 320 insertions(+), 133 deletions(-) create mode 100644 auditbeat/make.bat create mode 100644 dev-tools/mage/fields.go create mode 100644 generator/beat/{beat}/make.bat create mode 100644 generator/metricbeat/{beat}/make.bat create mode 100644 heartbeat/make.bat create mode 100644 libbeat/magefile.go create mode 100644 libbeat/make.bat create mode 100644 make.bat create mode 100644 packetbeat/make.bat diff --git a/CHANGELOG-developer.asciidoc b/CHANGELOG-developer.asciidoc index b47c53ca28a..cf844df59cf 100644 --- a/CHANGELOG-developer.asciidoc +++ b/CHANGELOG-developer.asciidoc @@ -25,6 +25,10 @@ The list below covers the major changes between 6.3.0 and master only. - Dashboards under _meta/kibana are expected to be decoded. See https://github.com/elastic/beats/pull/7224 for a conversion script. {pull}7265[7265] - Constructor `(github.com/elastic/beats/libbeat/output/codec/json).New` expects a new `escapeHTML` parameter. {pull}7445[7445] - Packaging has been refactored and updates are required. See the PR for migration details. {pull}7388[7388] +- `make fields` has been modified to use Mage (https://magefile.org/) in an effort to make + the building a Beat more cross-platform friendly (e.g. Windows). This requires that your Beat + has a magefile.go with a fields target. The `FIELDS_FILE_PATH` make variable is no longer + used because the value is specified in magefile.go. {pull}7670[7670] ==== Bugfixes diff --git a/auditbeat/Makefile b/auditbeat/Makefile index 36c51fbf0e6..08c0d4be4b7 100644 --- a/auditbeat/Makefile +++ b/auditbeat/Makefile @@ -6,7 +6,6 @@ GOX_OS?=linux windows ## @Building List of all OS to be supported by "make cross DEV_OS?=linux TESTING_ENVIRONMENT?=snapshot-noxpack ES_BEATS?=.. -FIELDS_FILE_PATH=module # Path to the libbeat Makefile include ${ES_BEATS}/libbeat/scripts/Makefile diff --git a/auditbeat/magefile.go b/auditbeat/magefile.go index 234afa2449f..0b45a4e52c6 100644 --- a/auditbeat/magefile.go +++ b/auditbeat/magefile.go @@ -91,6 +91,11 @@ func Update() error { return sh.Run("make", "update") } +// Fields generates a fields.yml for the Beat. +func Fields() error { + return mage.GenerateFieldsYAML("module") +} + // ----------------------------------------------------------------------------- // Customizations specific to Auditbeat. // - Config files are Go templates. diff --git a/auditbeat/make.bat b/auditbeat/make.bat new file mode 100644 index 00000000000..81de1ba946f --- /dev/null +++ b/auditbeat/make.bat @@ -0,0 +1,11 @@ +@echo off + +REM Windows wrapper for Mage (https://magefile.org/) that installs it +REM to %GOPATH%\bin from the Beats vendor directory. +REM +REM After running this once you may invoke mage.exe directly. + +WHERE mage +IF %ERRORLEVEL% NEQ 0 go install github.com/elastic/beats/vendor/github.com/magefile/mage + +mage %* diff --git a/dev-tools/jenkins_ci.ps1 b/dev-tools/jenkins_ci.ps1 index a27381a36e9..39abe9fad17 100755 --- a/dev-tools/jenkins_ci.ps1 +++ b/dev-tools/jenkins_ci.ps1 @@ -21,6 +21,14 @@ $env:PATH = "$env:GOPATH\bin;C:\tools\mingw64\bin;$env:PATH" # each run starts from a clean slate. $env:MAGEFILE_CACHE = "$env:WORKSPACE\.magefile" +exec { go install github.com/elastic/beats/vendor/github.com/magefile/mage } +exec { go get -u github.com/jstemmer/go-junit-report } + +echo "Building libbeat fields.yml" +cd libbeat +exec { mage fields } +cd .. + if (Test-Path "$env:beat") { cd "$env:beat" } else { @@ -35,20 +43,11 @@ New-Item -ItemType directory -Path build\coverage | Out-Null New-Item -ItemType directory -Path build\system-tests | Out-Null New-Item -ItemType directory -Path build\system-tests\run | Out-Null -exec { go get -u github.com/jstemmer/go-junit-report } +echo "Building fields.yml" +exec { mage fields } echo "Building $env:beat" -exec { go build } "Build FAILURE" - -# always build the libbeat fields -cp ..\libbeat\_meta\fields.common.yml ..\libbeat\_meta\fields.generated.yml -cat ..\libbeat\processors\*\_meta\fields.yml | Out-File -append -encoding UTF8 -filepath ..\libbeat\_meta\fields.generated.yml -cp ..\libbeat\_meta\fields.generated.yml ..\libbeat\fields.yml - -if ($env:beat -eq "metricbeat") { - cp .\_meta\fields.common.yml .\_meta\fields.generated.yml - python .\scripts\fields_collector.py | out-file -append -encoding UTF8 -filepath .\_meta\fields.generated.yml -} +exec { mage build } "Build FAILURE" echo "Unit testing $env:beat" go test -v $(go list ./... | select-string -Pattern "vendor" -NotMatch) 2>&1 | Out-File -encoding UTF8 build/TEST-go-unit.out diff --git a/dev-tools/mage/fields.go b/dev-tools/mage/fields.go new file mode 100644 index 00000000000..8f658483ffb --- /dev/null +++ b/dev-tools/mage/fields.go @@ -0,0 +1,48 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mage + +import ( + "path/filepath" + + "github.com/magefile/mage/sh" +) + +// GenerateFieldsYAML generates a fields.yml file for a Beat. This will include +// the common fields specified by libbeat, the common fields for the Beat, +// and any additional fields.yml files you specify. +// +// fieldsFiles specifies additional directories to search recursively for files +// named fields.yml. The contents of each fields.yml will be included in the +// generated file. +func GenerateFieldsYAML(fieldsFiles ...string) error { + const globalFieldsCmdPath = "libbeat/scripts/cmd/global_fields/main.go" + + beatsDir, err := ElasticBeatsDir() + if err != nil { + return err + } + + globalFieldsCmd := sh.RunCmd("go", "run", + filepath.Join(beatsDir, globalFieldsCmdPath), + "-es_beats_path", beatsDir, + "-beat_path", CWD(), + ) + + return globalFieldsCmd(fieldsFiles...) +} diff --git a/dev-tools/packaging/templates/common/magefile.go.tmpl b/dev-tools/packaging/templates/common/magefile.go.tmpl index 1a9822d2491..db7f7132110 100644 --- a/dev-tools/packaging/templates/common/magefile.go.tmpl +++ b/dev-tools/packaging/templates/common/magefile.go.tmpl @@ -70,3 +70,8 @@ func TestPackages() error { func Update() error { return sh.Run("make", "update") } + +// Fields generates a fields.yml for the Beat. +func Fields() error { + return mage.GenerateFieldsYAML("protos") +} diff --git a/filebeat/magefile.go b/filebeat/magefile.go index 8feb27a6908..41cb9ce9014 100644 --- a/filebeat/magefile.go +++ b/filebeat/magefile.go @@ -91,6 +91,11 @@ func Update() error { return sh.Run("make", "update") } +// Fields generates a fields.yml for the Beat. +func Fields() error { + return mage.GenerateFieldsYAML("module") +} + // ----------------------------------------------------------------------------- // Customizations specific to Filebeat. // - Include modules directory in packages (minus _meta and test files). diff --git a/filebeat/make.bat b/filebeat/make.bat index 3f92e367f42..81de1ba946f 100644 --- a/filebeat/make.bat +++ b/filebeat/make.bat @@ -1,7 +1,11 @@ -REM Batch script to build and test on Windows. You can use this in conjunction -REM with the Vagrant machine. -go build -go test ./... -go test -c -cover -covermode=count -coverpkg ./... -cd tests\system -nosetests +@echo off + +REM Windows wrapper for Mage (https://magefile.org/) that installs it +REM to %GOPATH%\bin from the Beats vendor directory. +REM +REM After running this once you may invoke mage.exe directly. + +WHERE mage +IF %ERRORLEVEL% NEQ 0 go install github.com/elastic/beats/vendor/github.com/magefile/mage + +mage %* diff --git a/generator/beat/Makefile b/generator/beat/Makefile index 0e5913ea44d..dd1e5be8557 100644 --- a/generator/beat/Makefile +++ b/generator/beat/Makefile @@ -1,4 +1 @@ -override FIELDS_FILE_PATH= -export FIELDS_FILE_PATH - include ../common/Makefile diff --git a/generator/beat/{beat}/Makefile b/generator/beat/{beat}/Makefile index e96c81728f1..b02b7c9afc1 100644 --- a/generator/beat/{beat}/Makefile +++ b/generator/beat/{beat}/Makefile @@ -13,7 +13,7 @@ MAGE_IMPORT_PATH=${BEAT_PATH}/vendor/github.com/magefile/mage # Initial beat setup .PHONY: setup -setup: copy-vendor update git-init +setup: copy-vendor git-init update git-add # Copy beats into vendor directory .PHONY: copy-vendor @@ -23,21 +23,18 @@ copy-vendor: rm -rf vendor/github.com/elastic/beats/.git vendor/github.com/elastic/beats/x-pack mkdir -p vendor/github.com/magefile cp -R ${BEAT_GOPATH}/src/github.com/elastic/beats/vendor/github.com/magefile/mage vendor/github.com/magefile + ls -la vendor/github.com/magefile + ls -la vendor/github.com/magefile/mage + ls -la vendor/github.com/magefile/mage/mage .PHONY: git-init git-init: git init - git add README.md CONTRIBUTING.md - git commit -m "Initial commit" - git add LICENSE.txt - git commit -m "Add the LICENSE" - git add .gitignore - git commit -m "Add git settings" - git add . - git reset -- .travis.yml - git commit -m "Add {beat}" - git add .travis.yml - git commit -m "Add Travis CI" + +.PHONY: git-add +git-add: + git add -A + git commit -m "Add generated {beat} files" # Collects all dependencies and then calls update .PHONY: collect diff --git a/generator/beat/{beat}/magefile.go b/generator/beat/{beat}/magefile.go index 545af4c3d10..13c6184b5fa 100644 --- a/generator/beat/{beat}/magefile.go +++ b/generator/beat/{beat}/magefile.go @@ -89,3 +89,8 @@ func TestPackages() error { func Update() error { return sh.Run("make", "update") } + +// Fields generates a fields.yml for the Beat. +func Fields() error { + return mage.GenerateFieldsYAML() +} diff --git a/generator/beat/{beat}/make.bat b/generator/beat/{beat}/make.bat new file mode 100644 index 00000000000..72de5798dfa --- /dev/null +++ b/generator/beat/{beat}/make.bat @@ -0,0 +1,11 @@ +@echo off + +REM Windows wrapper for Mage (https://magefile.org/) that installs it +REM to %GOPATH%\bin from the Beats vendor directory. +REM +REM After running this once you may invoke mage.exe directly. + +WHERE mage +IF %ERRORLEVEL% NEQ 0 go install {beat_path}/vendor/github.com/magefile/mage + +mage %* diff --git a/generator/common/Makefile b/generator/common/Makefile index 248b6c7119c..8ad037be6e0 100644 --- a/generator/common/Makefile +++ b/generator/common/Makefile @@ -11,23 +11,36 @@ PREPARE_COMMAND?= test: prepare-test . ${PYTHON_ENV}/bin/activate; \ export GOPATH=${PWD}/build ; \ - export PATH=${PATH}:${PWD}/build/bin; \ + export PATH=$${GOPATH}/bin:${PATH}; \ cd ${BEAT_PATH} ; \ $(MAKE) copy-vendor || exit 1 ; \ ${PREPARE_COMMAND} \ + $(MAKE) git-init || exit 1 ; \ $(MAKE) update || exit 1 ; \ + $(MAKE) git-add || exit 1 ; \ $(MAKE) check CHECK_HEADERS_DISABLED=y || exit 1 ; \ $(MAKE) || exit 1 ; \ $(MAKE) unit +.PHONY: prepare-test prepare-test:: python-env # Makes sure to use current version of beats for testing mkdir -p ${BUILD_DIR}/src/github.com/elastic/beats/ - rsync -a --exclude=build ${PWD}/../../* ${BUILD_DIR}/src/github.com/elastic/beats/ + rsync -a \ + --include=vendor/github.com/magefile/mage/build \ + --exclude=build/ \ + --exclude=.git/ \ + ${PWD}/../../* ${BUILD_DIR}/src/github.com/elastic/beats/ mkdir -p ${BEAT_PATH} export GOPATH=${PWD}/build ; \ - . ${PYTHON_ENV}/bin/activate && python ${PWD}/build/src/github.com/elastic/beats/script/generate.py --type=${BEAT_TYPE} --project_name=Testbeat --github_name=ruflin --beat_path=beatpath/testbeat --full_name="Nicolas Ruflin" + . ${PYTHON_ENV}/bin/activate && \ + python ${PWD}/build/src/github.com/elastic/beats/script/generate.py \ + --type=${BEAT_TYPE} \ + --project_name=Testbeat \ + --github_name=ruflin \ + --beat_path=beatpath/testbeat \ + --full_name="Nicolas Ruflin" # Runs test build for the created beat .PHONY: test-build diff --git a/generator/metricbeat/Makefile b/generator/metricbeat/Makefile index 11203d15e0f..f91acb876a3 100644 --- a/generator/metricbeat/Makefile +++ b/generator/metricbeat/Makefile @@ -1,16 +1,14 @@ -BEAT_TYPE=metricbeat -PREPARE_COMMAND=MODULE=elastic METRICSET=test make create-metricset ; FIELDS_FILE_PATH=module -override FIELDS_FILE_PATH= -export FIELDS_FILE_PATH - +BEAT_TYPE = metricbeat +PREPARE_COMMAND = MODULE=elastic METRICSET=test make create-metricset; include ../common/Makefile +.PHONY: prepare-test prepare-test:: python-env - mkdir -p ${BEAT_PATH}/scripts rsync -a --exclude=build ${PWD}/../../metricbeat/scripts/generate_imports_helper.py ${BEAT_PATH}/scripts # Collects all dependencies and then calls update .PHONY: collect collect: fields + diff --git a/generator/metricbeat/{beat}/Makefile b/generator/metricbeat/{beat}/Makefile index 9882c9783f0..478a3f7a4e1 100644 --- a/generator/metricbeat/{beat}/Makefile +++ b/generator/metricbeat/{beat}/Makefile @@ -13,7 +13,7 @@ MAGE_IMPORT_PATH=${BEAT_PATH}/vendor/github.com/magefile/mage # Initial beat setup .PHONY: setup -setup: copy-vendor create-metricset collect git-init +setup: copy-vendor git-init create-metricset collect git-add # Copy beats into vendor directory .PHONY: copy-vendor @@ -28,5 +28,8 @@ copy-vendor: .PHONY: git-init git-init: git init - git add --all - git commit -m 'Initial add by Beat generator' + +.PHONY: git-add +git-add: + git add -A + git commit -m "Add generated {beat} files" diff --git a/generator/metricbeat/{beat}/magefile.go b/generator/metricbeat/{beat}/magefile.go index 545af4c3d10..fa75eff3a80 100644 --- a/generator/metricbeat/{beat}/magefile.go +++ b/generator/metricbeat/{beat}/magefile.go @@ -89,3 +89,8 @@ func TestPackages() error { func Update() error { return sh.Run("make", "update") } + +// Fields generates a fields.yml for the Beat. +func Fields() error { + return mage.GenerateFieldsYAML("module") +} diff --git a/generator/metricbeat/{beat}/make.bat b/generator/metricbeat/{beat}/make.bat new file mode 100644 index 00000000000..72de5798dfa --- /dev/null +++ b/generator/metricbeat/{beat}/make.bat @@ -0,0 +1,11 @@ +@echo off + +REM Windows wrapper for Mage (https://magefile.org/) that installs it +REM to %GOPATH%\bin from the Beats vendor directory. +REM +REM After running this once you may invoke mage.exe directly. + +WHERE mage +IF %ERRORLEVEL% NEQ 0 go install {beat_path}/vendor/github.com/magefile/mage + +mage %* diff --git a/heartbeat/Makefile b/heartbeat/Makefile index 16053181ce5..498c10747b5 100644 --- a/heartbeat/Makefile +++ b/heartbeat/Makefile @@ -2,7 +2,6 @@ BEAT_NAME=heartbeat BEAT_TITLE=Heartbeat SYSTEM_TESTS=true TEST_ENVIRONMENT=false -FIELDS_FILE_PATH=monitors/active # Path to the libbeat Makefile -include ../libbeat/scripts/Makefile diff --git a/heartbeat/magefile.go b/heartbeat/magefile.go index f52d0935da9..a23b8687e00 100644 --- a/heartbeat/magefile.go +++ b/heartbeat/magefile.go @@ -88,3 +88,8 @@ func TestPackages() error { func Update() error { return sh.Run("make", "update") } + +// Fields generates a fields.yml for the Beat. +func Fields() error { + return mage.GenerateFieldsYAML("monitors/active") +} diff --git a/heartbeat/make.bat b/heartbeat/make.bat new file mode 100644 index 00000000000..81de1ba946f --- /dev/null +++ b/heartbeat/make.bat @@ -0,0 +1,11 @@ +@echo off + +REM Windows wrapper for Mage (https://magefile.org/) that installs it +REM to %GOPATH%\bin from the Beats vendor directory. +REM +REM After running this once you may invoke mage.exe directly. + +WHERE mage +IF %ERRORLEVEL% NEQ 0 go install github.com/elastic/beats/vendor/github.com/magefile/mage + +mage %* diff --git a/libbeat/Makefile b/libbeat/Makefile index d63820feabc..93b34f77104 100644 --- a/libbeat/Makefile +++ b/libbeat/Makefile @@ -1,7 +1,6 @@ BEAT_NAME=libbeat TEST_ENVIRONMENT?=true SYSTEM_TESTS=true -FIELDS_FILE_PATH=processors include scripts/Makefile diff --git a/libbeat/generator/fields/fields.go b/libbeat/generator/fields/fields.go index 68365dced93..02ec5374948 100644 --- a/libbeat/generator/fields/fields.go +++ b/libbeat/generator/fields/fields.go @@ -20,12 +20,13 @@ package fields import ( "bufio" "bytes" - "fmt" "io/ioutil" "os" "path" "path/filepath" "strings" + + "github.com/pkg/errors" ) var ( @@ -152,7 +153,7 @@ func createIfNotExists(inPath, outPath string) error { if os.IsNotExist(err) { err := copyFileWithFlag(inPath, outPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC) if err != nil { - fmt.Println("Cannot find _meta/fields.yml") + return err } return nil } @@ -162,12 +163,17 @@ func createIfNotExists(inPath, outPath string) error { func copyFileWithFlag(in, out string, flag int) error { input, err := ioutil.ReadFile(in) if err != nil { - return err + return errors.Wrap(err, "failed to read source in copy") + } + + if err := os.MkdirAll(filepath.Dir(out), 0755); err != nil { + return errors.Wrapf(err, "failed to create destination dir for copy "+ + "at %v", filepath.Dir(out)) } output, err := os.OpenFile(out, flag, 0644) if err != nil { - return err + return errors.Wrap(err, "failed to open destination file for copy") } defer output.Close() diff --git a/libbeat/magefile.go b/libbeat/magefile.go new file mode 100644 index 00000000000..939eac4adb0 --- /dev/null +++ b/libbeat/magefile.go @@ -0,0 +1,39 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build mage + +package main + +import ( + "github.com/elastic/beats/dev-tools/mage" +) + +// Build builds the Beat binary. +func Build() error { + return mage.Build(mage.DefaultBuildArgs()) +} + +// Clean cleans all generated files and build artifacts. +func Clean() error { + return mage.Clean() +} + +// Fields generates a fields.yml for the Beat. +func Fields() error { + return mage.GenerateFieldsYAML("processors") +} diff --git a/libbeat/make.bat b/libbeat/make.bat new file mode 100644 index 00000000000..81de1ba946f --- /dev/null +++ b/libbeat/make.bat @@ -0,0 +1,11 @@ +@echo off + +REM Windows wrapper for Mage (https://magefile.org/) that installs it +REM to %GOPATH%\bin from the Beats vendor directory. +REM +REM After running this once you may invoke mage.exe directly. + +WHERE mage +IF %ERRORLEVEL% NEQ 0 go install github.com/elastic/beats/vendor/github.com/magefile/mage + +mage %* diff --git a/libbeat/scripts/Makefile b/libbeat/scripts/Makefile index d2b109cbcbb..8e8a0ac3887 100755 --- a/libbeat/scripts/Makefile +++ b/libbeat/scripts/Makefile @@ -20,7 +20,9 @@ ELASTIC_LICENSE_FILE?=../licenses/ELASTIC-LICENSE.txt SECCOMP_BINARY?=${BEAT_NAME} SECCOMP_BLACKLIST?=${ES_BEATS}/libbeat/common/seccomp/seccomp-profiler-blacklist.txt SECCOMP_ALLOWLIST?=${ES_BEATS}/libbeat/common/seccomp/seccomp-profiler-allow.txt +MAGE_PRESENT := $(shell command -v mage 2> /dev/null) MAGE_IMPORT_PATH?=github.com/elastic/beats/vendor/github.com/magefile/mage +export MAGE_IMPORT_PATH space:=$() # comma:=, @@ -304,9 +306,8 @@ coverage-report: .PHONY: fields -fields: - @go run ${ES_BEATS}/libbeat/scripts/cmd/global_fields/main.go --es_beats_path $(ES_BEATS) --beat_path $(PWD) $(FIELDS_FILE_PATH) - +fields: mage + mage -v fields .PHONY: libbeat_fields libbeat_fields: @@ -442,7 +443,9 @@ seccomp-package: .PHONY: mage mage: - @go install ${MAGE_IMPORT_PATH} +ifndef MAGE_PRESENT + go install ${MAGE_IMPORT_PATH} +endif .PHONY: release release: mage diff --git a/libbeat/scripts/cmd/global_fields/main.go b/libbeat/scripts/cmd/global_fields/main.go index 8bd5b691506..74fc94a603d 100644 --- a/libbeat/scripts/cmd/global_fields/main.go +++ b/libbeat/scripts/cmd/global_fields/main.go @@ -31,41 +31,46 @@ func main() { beatPath := flag.String("beat_path", ".", "Path to your Beat") flag.Parse() - beatFieldsPath := flag.Args() + beatFieldsPaths := flag.Args() name := filepath.Base(*beatPath) - err := os.MkdirAll(filepath.Join(*beatPath, "_meta"), 0744) + if *beatPath == "" { + fmt.Fprintf(os.Stderr, "beat_path cannot be empty") + os.Exit(1) + } + + err := os.MkdirAll(filepath.Join(*beatPath, "_meta"), 0755) if err != nil { - fmt.Printf("Cannot create _meta dir for %s: %v\n", name, err) + fmt.Fprintf(os.Stderr, "Cannot create _meta dir for %s: %+v\n", name, err) os.Exit(1) } - if len(beatFieldsPath) == 0 { + if len(beatFieldsPaths) == 0 { fmt.Println("No field files to collect") err = fields.AppendFromLibbeat(*esBeatsPath, *beatPath) if err != nil { - fmt.Printf("Cannot generate global fields.yml for %s: %v\n", name, err) + fmt.Fprintf(os.Stderr, "Cannot generate global fields.yml for %s: %+v\n", name, err) os.Exit(2) } return } - if *beatPath == "" { - fmt.Println("beat_path cannot be empty") - os.Exit(1) - } + var fieldsFiles []*fields.YmlFile + for _, fieldsFilePath := range beatFieldsPaths { + pathToModules := filepath.Join(*beatPath, fieldsFilePath) - pathToModules := filepath.Join(*beatPath, beatFieldsPath[0]) - fieldFiles, err := fields.CollectModuleFiles(pathToModules) - if err != nil { - fmt.Printf("Cannot collect fields.yml files: %v\n", err) - os.Exit(2) + fieldsFile, err := fields.CollectModuleFiles(pathToModules) + if err != nil { + fmt.Fprintf(os.Stderr, "Cannot collect fields.yml files: %+v\n", err) + os.Exit(2) + } + fieldsFiles = append(fieldsFiles, fieldsFile...) } - err = fields.Generate(*esBeatsPath, *beatPath, fieldFiles) + err = fields.Generate(*esBeatsPath, *beatPath, fieldsFiles) if err != nil { - fmt.Printf("Cannot generate global fields.yml file for %s: %v\n", name, err) + fmt.Fprintf(os.Stderr, "Cannot generate global fields.yml file for %s: %+v\n", name, err) os.Exit(3) } diff --git a/make.bat b/make.bat new file mode 100644 index 00000000000..81de1ba946f --- /dev/null +++ b/make.bat @@ -0,0 +1,11 @@ +@echo off + +REM Windows wrapper for Mage (https://magefile.org/) that installs it +REM to %GOPATH%\bin from the Beats vendor directory. +REM +REM After running this once you may invoke mage.exe directly. + +WHERE mage +IF %ERRORLEVEL% NEQ 0 go install github.com/elastic/beats/vendor/github.com/magefile/mage + +mage %* diff --git a/metricbeat/Makefile b/metricbeat/Makefile index a055c1a6f04..f42f8cca6f4 100644 --- a/metricbeat/Makefile +++ b/metricbeat/Makefile @@ -4,7 +4,6 @@ BEAT_TITLE?=Metricbeat SYSTEM_TESTS?=true TEST_ENVIRONMENT?=true ES_BEATS?=.. -FIELDS_FILE_PATH?=module # Metricbeat can only be cross-compiled on platforms not requiring CGO. GOX_OS=netbsd linux windows diff --git a/metricbeat/magefile.go b/metricbeat/magefile.go index 02f9bbe516c..c19cbf578a9 100644 --- a/metricbeat/magefile.go +++ b/metricbeat/magefile.go @@ -91,6 +91,11 @@ func Update() error { return sh.Run("make", "update") } +// Fields generates a fields.yml for the Beat. +func Fields() error { + return mage.GenerateFieldsYAML("module") +} + // ----------------------------------------------------------------------------- // Customizations specific to Metricbeat. // - Include modules.d directory in packages. diff --git a/metricbeat/make.bat b/metricbeat/make.bat index 7edecf7aaca..81de1ba946f 100644 --- a/metricbeat/make.bat +++ b/metricbeat/make.bat @@ -1,34 +1,11 @@ @echo off +REM Windows wrapper for Mage (https://magefile.org/) that installs it +REM to %GOPATH%\bin from the Beats vendor directory. REM -REM Batch script to build and test on Windows. You can use this in conjunction -REM with the Vagrant machine. -REM - -go install github.com/elastic/beats/vendor/github.com/pierrre/gotestcover -if %errorlevel% neq 0 exit /b %errorlevel% - -echo Building -go build -if %errorlevel% neq 0 exit /b %errorlevel% - -echo Testing -mkdir build\coverage -gotestcover -race -coverprofile=build/coverage/integration.cov github.com/elastic/beats/metricbeat/... -if %errorlevel% neq 0 exit /b %errorlevel% - -echo System Testing -go test -c -covermode=atomic -coverpkg ./... -if %errorlevel% neq 0 exit /b %errorlevel% -nosetests -v -w tests\system --process-timeout=30 -if %errorlevel% neq 0 exit /b %errorlevel% +REM After running this once you may invoke mage.exe directly. -echo Aggregating Coverage Reports -python ..\dev-tools\aggregate_coverage.py -o build\coverage\system.cov .\build\system-tests\run -if %errorlevel% neq 0 exit /b %errorlevel% -python ..\dev-tools\aggregate_coverage.py -o build\coverage\full.cov .\build\coverage -if %errorlevel% neq 0 exit /b %errorlevel% -go tool cover -html=build\coverage\full.cov -o build\coverage\full.html -if %errorlevel% neq 0 exit /b %errorlevel% +WHERE mage +IF %ERRORLEVEL% NEQ 0 go install github.com/elastic/beats/vendor/github.com/magefile/mage -echo Success +mage %* diff --git a/packetbeat/Makefile b/packetbeat/Makefile index 7c0b5325834..03ed18219db 100644 --- a/packetbeat/Makefile +++ b/packetbeat/Makefile @@ -3,7 +3,6 @@ BEAT_TITLE?=Packetbeat SYSTEM_TESTS?=true TEST_ENVIRONMENT=false ES_BEATS?=.. -FIELDS_FILE_PATH=protos include ${ES_BEATS}/libbeat/scripts/Makefile diff --git a/packetbeat/magefile.go b/packetbeat/magefile.go index 98c37ea5167..b002c88e69d 100644 --- a/packetbeat/magefile.go +++ b/packetbeat/magefile.go @@ -118,6 +118,11 @@ func Update() error { return sh.Run("make", "update") } +// Fields generates a fields.yml for the Beat. +func Fields() error { + return mage.GenerateFieldsYAML("protos") +} + // ----------------------------------------------------------------------------- // Customizations specific to Packetbeat. // - Config file contains an OS specific device name (affects darwin, windows). diff --git a/packetbeat/make.bat b/packetbeat/make.bat new file mode 100644 index 00000000000..81de1ba946f --- /dev/null +++ b/packetbeat/make.bat @@ -0,0 +1,11 @@ +@echo off + +REM Windows wrapper for Mage (https://magefile.org/) that installs it +REM to %GOPATH%\bin from the Beats vendor directory. +REM +REM After running this once you may invoke mage.exe directly. + +WHERE mage +IF %ERRORLEVEL% NEQ 0 go install github.com/elastic/beats/vendor/github.com/magefile/mage + +mage %* diff --git a/winlogbeat/magefile.go b/winlogbeat/magefile.go index 5500f9fdca8..1d8ad837d83 100644 --- a/winlogbeat/magefile.go +++ b/winlogbeat/magefile.go @@ -88,3 +88,8 @@ func TestPackages() error { func Update() error { return sh.Run("make", "update") } + +// Fields generates a fields.yml for the Beat. +func Fields() error { + return mage.GenerateFieldsYAML() +} diff --git a/winlogbeat/make.bat b/winlogbeat/make.bat index 0a46f8949b6..81de1ba946f 100644 --- a/winlogbeat/make.bat +++ b/winlogbeat/make.bat @@ -1,34 +1,11 @@ @echo off +REM Windows wrapper for Mage (https://magefile.org/) that installs it +REM to %GOPATH%\bin from the Beats vendor directory. REM -REM Batch script to build and test on Windows. You can use this in conjunction -REM with the Vagrant machine. -REM - -go install github.com/elastic/beats/vendor/github.com/pierrre/gotestcover -if %errorlevel% neq 0 exit /b %errorlevel% - -echo Building -go build -if %errorlevel% neq 0 exit /b %errorlevel% - -echo Testing -mkdir build\coverage -gotestcover -race -coverprofile=build/coverage/integration.cov github.com/elastic/beats/winlogbeat/... -if %errorlevel% neq 0 exit /b %errorlevel% - -echo System Testing -go test -c -covermode=atomic -coverpkg ./... -if %errorlevel% neq 0 exit /b %errorlevel% -nosetests -v -w tests\system --process-timeout=30 -if %errorlevel% neq 0 exit /b %errorlevel% +REM After running this once you may invoke mage.exe directly. -echo Aggregating Coverage Reports -python ..\dev-tools\aggregate_coverage.py -o build\coverage\system.cov .\build\system-tests\run -if %errorlevel% neq 0 exit /b %errorlevel% -python ..\dev-tools\aggregate_coverage.py -o build\coverage\full.cov .\build\coverage -if %errorlevel% neq 0 exit /b %errorlevel% -go tool cover -html=build\coverage\full.cov -o build\coverage\full.html -if %errorlevel% neq 0 exit /b %errorlevel% +WHERE mage +IF %ERRORLEVEL% NEQ 0 go install github.com/elastic/beats/vendor/github.com/magefile/mage -echo Success +mage %* From 56f265c1f7bc675a5339330e4558783c933fe91e Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Mon, 23 Jul 2018 11:45:34 -0400 Subject: [PATCH 2/2] Remove debugging commands --- generator/beat/{beat}/Makefile | 3 --- 1 file changed, 3 deletions(-) diff --git a/generator/beat/{beat}/Makefile b/generator/beat/{beat}/Makefile index b02b7c9afc1..8fdde0a7e0c 100644 --- a/generator/beat/{beat}/Makefile +++ b/generator/beat/{beat}/Makefile @@ -23,9 +23,6 @@ copy-vendor: rm -rf vendor/github.com/elastic/beats/.git vendor/github.com/elastic/beats/x-pack mkdir -p vendor/github.com/magefile cp -R ${BEAT_GOPATH}/src/github.com/elastic/beats/vendor/github.com/magefile/mage vendor/github.com/magefile - ls -la vendor/github.com/magefile - ls -la vendor/github.com/magefile/mage - ls -la vendor/github.com/magefile/mage/mage .PHONY: git-init git-init: