From 36e879749efd2f9ad38eb49a5115b9120d69b8ae Mon Sep 17 00:00:00 2001 From: Chris Mark Date: Fri, 15 Nov 2019 00:03:27 +0200 Subject: [PATCH] Replace generate_imports python script with mage imports (#14498) (cherry picked from commit 6b46c296b0aa09a85cac780b566747cec50b6b7a) --- .../module_include_list.go | 40 ++++++++---- dev-tools/mage/fields.go | 39 +++++++++-- filebeat/magefile.go | 4 +- generator/common/beatgen/beatgen.go | 3 +- generator/common/beatgen/setup/setup.go | 12 +--- generator/metricbeat/{beat}/magefile.go | 6 ++ heartbeat/Makefile | 4 +- heartbeat/magefile.go | 10 +++ heartbeat/monitors/defaults/default.go | 9 +-- metricbeat/Makefile | 4 +- metricbeat/include/list_common.go | 9 +-- metricbeat/include/list_docker.go | 12 +--- metricbeat/magefile.go | 6 ++ metricbeat/scripts/generate_imports_helper.py | 65 ------------------- metricbeat/scripts/mage/fields.go | 52 +++++++++++++++ packetbeat/magefile.go | 5 +- script/generate_imports.py | 63 ------------------ x-pack/filebeat/magefile.go | 4 +- 18 files changed, 158 insertions(+), 189 deletions(-) delete mode 100644 metricbeat/scripts/generate_imports_helper.py create mode 100644 metricbeat/scripts/mage/fields.go delete mode 100644 script/generate_imports.py diff --git a/dev-tools/cmd/module_include_list/module_include_list.go b/dev-tools/cmd/module_include_list/module_include_list.go index 36f93c0abdea..d3e5d3c2b3e7 100644 --- a/dev-tools/cmd/module_include_list/module_include_list.go +++ b/dev-tools/cmd/module_include_list/module_include_list.go @@ -50,18 +50,22 @@ Options: `[1:] var ( - license string - pkg string - outFile string - moduleDirs stringSliceFlag - importDirs stringSliceFlag + license string + pkg string + outFile string + buildTags string + moduleDirs stringSliceFlag + moduleExcludeDirs stringSliceFlag + importDirs stringSliceFlag ) func init() { flag.StringVar(&license, "license", "ASL2", "License header for generated file (ASL2 or Elastic).") flag.StringVar(&pkg, "pkg", "include", "Package name.") flag.StringVar(&outFile, "out", "include/list.go", "Output file.") + flag.StringVar(&buildTags, "buildTags", "", "Build Tags.") flag.Var(&moduleDirs, "moduleDir", "Directory to search for modules to include") + flag.Var(&moduleExcludeDirs, "moduleExcludeDirs", "Directory to exclude from the list") flag.Var(&importDirs, "import", "Directory to include") flag.Usage = usageFlag } @@ -138,9 +142,10 @@ func main() { // Populate the template. var buf bytes.Buffer err = Template.Execute(&buf, Data{ - License: license, - Package: pkg, - Imports: imports, + License: license, + Package: pkg, + BuildTags: buildTags, + Imports: imports, }) if err != nil { log.Fatalf("Failed executing template: %v", err) @@ -168,7 +173,7 @@ var Template = template.Must(template.New("normalizations").Funcs(map[string]int {{ .License | trim }} // Code generated by beats/dev-tools/cmd/module_include_list/module_include_list.go - DO NOT EDIT. - +{{ .BuildTags }} package {{ .Package }} import ( @@ -180,9 +185,10 @@ import ( `[1:])) type Data struct { - License string - Package string - Imports []string + License string + Package string + BuildTags string + Imports []string } //stringSliceFlag is a flag type that allows more than one value to be specified. @@ -211,6 +217,16 @@ func findModuleAndDatasets() ([]string, error) { for _, metaDir := range metaDirs { // Strip off _meta. + skipDir := false + for _, excludeModule := range moduleExcludeDirs { + if strings.Contains(metaDir, excludeModule) { + skipDir = true + break + } + } + if skipDir { + continue + } dirs = append(dirs, filepath.Dir(metaDir)) } } diff --git a/dev-tools/mage/fields.go b/dev-tools/mage/fields.go index bc4f6cee68ff..009d720840cb 100644 --- a/dev-tools/mage/fields.go +++ b/dev-tools/mage/fields.go @@ -37,6 +37,28 @@ const ( FieldsAllYML = "build/fields/fields.all.yml" ) +// IncludeListOptions stores the options for IncludeList generation +type IncludeListOptions struct { + ImportDirs []string + ModuleDirs []string + ModulesToExclude []string + Outfile string + BuildTags string + Pkg string +} + +// DefaultIncludeListOptions initializes IncludeListOptions struct with default values +func DefaultIncludeListOptions() IncludeListOptions { + return IncludeListOptions{ + ImportDirs: nil, + ModuleDirs: []string{"module"}, + ModulesToExclude: nil, + Outfile: "include/list.go", + BuildTags: "", + Pkg: "include", + } +} + // FieldsBuilder is the interface projects to implement for building field data. type FieldsBuilder interface { // Generate all fields.go files. @@ -143,13 +165,13 @@ func GenerateModuleFieldsGo(moduleDir string) error { // GenerateModuleIncludeListGo generates an include/list.go file containing // a import statement for each module and dataset. func GenerateModuleIncludeListGo() error { - return GenerateIncludeListGo(nil, []string{"module"}) + return GenerateIncludeListGo(DefaultIncludeListOptions()) } // GenerateIncludeListGo generates an include/list.go file containing imports // for the packages that match the paths (or globs) in importDirs (optional) // and moduleDirs (optional). -func GenerateIncludeListGo(importDirs []string, moduleDirs []string) error { +func GenerateIncludeListGo(options IncludeListOptions) error { const moduleIncludeListCmdPath = "dev-tools/cmd/module_include_list/module_include_list.go" beatsDir, err := ElasticBeatsDir() @@ -160,22 +182,29 @@ func GenerateIncludeListGo(importDirs []string, moduleDirs []string) error { includeListCmd := sh.RunCmd("go", "run", filepath.Join(beatsDir, moduleIncludeListCmdPath), "-license", toLibbeatLicenseName(BeatLicense), + "-out", options.Outfile, "-buildTags", options.BuildTags, + "-pkg", options.Pkg, ) var args []string - for _, dir := range importDirs { + for _, dir := range options.ImportDirs { if !filepath.IsAbs(dir) { dir = CWD(dir) } args = append(args, "-import", dir) } - for _, dir := range moduleDirs { + for _, dir := range options.ModuleDirs { if !filepath.IsAbs(dir) { dir = CWD(dir) } args = append(args, "-moduleDir", dir) } - + for _, dir := range options.ModulesToExclude { + if !filepath.IsAbs(dir) { + dir = CWD(dir) + } + args = append(args, "-moduleExcludeDirs", dir) + } return includeListCmd(args...) } diff --git a/filebeat/magefile.go b/filebeat/magefile.go index 864c3fac316a..efc4d8afad8c 100644 --- a/filebeat/magefile.go +++ b/filebeat/magefile.go @@ -112,7 +112,9 @@ func configYML() error { // includeList generates include/list.go with imports for inputs. func includeList() error { - return devtools.GenerateIncludeListGo([]string{"input/*"}, []string{"module"}) + options := devtools.DefaultIncludeListOptions() + options.ImportDirs = []string{"input/*"} + return devtools.GenerateIncludeListGo(options) } // Fields generates fields.yml and fields.go files for the Beat. diff --git a/generator/common/beatgen/beatgen.go b/generator/common/beatgen/beatgen.go index cdb3bd1e5c9c..07a8ed09489b 100644 --- a/generator/common/beatgen/beatgen.go +++ b/generator/common/beatgen/beatgen.go @@ -129,8 +129,7 @@ func VendorUpdate() error { } devtools.SetElasticBeatsDir(getAbsoluteBeatsPath()) - mg.SerialDeps(setup.CopyVendor, setup.LinkImportsHelper) - return nil + return setup.CopyVendor() } // returns a "compleated" config object with everything we need diff --git a/generator/common/beatgen/setup/setup.go b/generator/common/beatgen/setup/setup.go index 7bebb0a735a4..5e5d6b96ced8 100644 --- a/generator/common/beatgen/setup/setup.go +++ b/generator/common/beatgen/setup/setup.go @@ -48,17 +48,7 @@ func RunSetup() error { if err != nil { return errors.Wrapf(err, "error copying pkg to %s", vendorPath) } - return LinkImportsHelper() -} - -// LinkImportsHelper links generate_imports_helper.py -func LinkImportsHelper() error { - vendorPath := "./vendor/github.com/" - pwd, err := os.Getwd() - if err != nil { - return errors.Wrap(err, "error gettting current directory") - } - return sh.Run("ln", "-sf", filepath.Join(pwd, vendorPath, "elastic/beats/metricbeat/scripts/generate_imports_helper.py"), filepath.Join(pwd, vendorPath, "elastic/beats/script/generate_imports_helper.py")) + return nil } // CopyVendor copies a new version of beats into the vendor directory of PWD diff --git a/generator/metricbeat/{beat}/magefile.go b/generator/metricbeat/{beat}/magefile.go index cbd0cbc1522d..5b1521a1c806 100644 --- a/generator/metricbeat/{beat}/magefile.go +++ b/generator/metricbeat/{beat}/magefile.go @@ -104,6 +104,12 @@ func Update() error { return update.Update() } +// Imports generates an include/list.go file containing +// a import statement for each module and dataset. +func Imports() error { + return devtools.GenerateModuleIncludeListGo() +} + // Test runs all available tests func Test() { mg.Deps(unittest.GoUnitTest) diff --git a/heartbeat/Makefile b/heartbeat/Makefile index dde71c063127..80e4650cd161 100644 --- a/heartbeat/Makefile +++ b/heartbeat/Makefile @@ -12,9 +12,9 @@ collect: imports kibana # Generate imports for all monitors .PHONY: imports -imports: python-env +imports: @mkdir -p include - @${PYTHON_ENV}/bin/python ${ES_BEATS}/script/generate_imports.py --out monitors/defaults/default.go ${BEAT_PATH} + mage imports # Collects all module dashboards .PHONY: kibana diff --git a/heartbeat/magefile.go b/heartbeat/magefile.go index 7aafc45cf23b..e9524309f372 100644 --- a/heartbeat/magefile.go +++ b/heartbeat/magefile.go @@ -110,6 +110,16 @@ func Fields() error { return devtools.GenerateFieldsYAML("monitors/active") } +// Imports generates an include/list.go file containing +// a import statement for each module and dataset. +func Imports() error { + options := devtools.DefaultIncludeListOptions() + options.ModuleDirs = []string{"monitors"} + options.Outfile = "monitors/defaults/default.go" + options.Pkg = "defaults" + return devtools.GenerateIncludeListGo(options) +} + // GoTestUnit executes the Go unit tests. // Use TEST_COVERAGE=true to enable code coverage profiling. // Use RACE_DETECTOR=true to enable the race detector. diff --git a/heartbeat/monitors/defaults/default.go b/heartbeat/monitors/defaults/default.go index 35bb76fb9514..5e65d46fab3e 100644 --- a/heartbeat/monitors/defaults/default.go +++ b/heartbeat/monitors/defaults/default.go @@ -15,17 +15,12 @@ // specific language governing permissions and limitations // under the License. -// Code generated by 'make imports' - DO NOT EDIT. +// Code generated by beats/dev-tools/cmd/module_include_list/module_include_list.go - DO NOT EDIT. -/* -Package defaults imports all Monitor packages so that they -register with the global monitor registry. This package can be imported in the -main package to automatically register all of the standard supported Heartbeat -modules. -*/ package defaults import ( + // Import packages that need to register themselves. _ "github.com/elastic/beats/heartbeat/monitors/active/http" _ "github.com/elastic/beats/heartbeat/monitors/active/icmp" _ "github.com/elastic/beats/heartbeat/monitors/active/tcp" diff --git a/metricbeat/Makefile b/metricbeat/Makefile index 72567d2d478e..05b4227788c2 100644 --- a/metricbeat/Makefile +++ b/metricbeat/Makefile @@ -41,9 +41,9 @@ configs: python-env # Generates imports for all modules and metricsets .PHONY: imports -imports: python-env +imports: @mkdir -p include - @${PYTHON_ENV}/bin/python ${ES_BEATS}/script/generate_imports.py ${BEAT_PATH} + mage imports # Runs all collection steps and updates afterwards .PHONY: collect diff --git a/metricbeat/include/list_common.go b/metricbeat/include/list_common.go index b5158ec8380a..a655dea57f33 100644 --- a/metricbeat/include/list_common.go +++ b/metricbeat/include/list_common.go @@ -15,17 +15,12 @@ // specific language governing permissions and limitations // under the License. -// Code generated by 'make imports' - DO NOT EDIT. +// Code generated by beats/dev-tools/cmd/module_include_list/module_include_list.go - DO NOT EDIT. -/* -Package include imports all Module and MetricSet packages so that they register -their factories with the global registry. This package can be imported in the -main package to automatically register all of the standard supported Metricbeat -modules. -*/ package include import ( + // Import packages that need to register themselves. _ "github.com/elastic/beats/metricbeat/module/aerospike" _ "github.com/elastic/beats/metricbeat/module/aerospike/namespace" _ "github.com/elastic/beats/metricbeat/module/apache" diff --git a/metricbeat/include/list_docker.go b/metricbeat/include/list_docker.go index 1fb90c3219cc..5a61d9a92535 100644 --- a/metricbeat/include/list_docker.go +++ b/metricbeat/include/list_docker.go @@ -15,20 +15,14 @@ // specific language governing permissions and limitations // under the License. -// Code generated by 'make imports' - DO NOT EDIT. +// Code generated by beats/dev-tools/cmd/module_include_list/module_include_list.go - DO NOT EDIT. // +build linux darwin windows -/* -Package include imports all Module and MetricSet packages so that they register -their factories with the global registry. This package can be imported in the -main package to automatically register all of the standard supported Metricbeat -modules. -*/ package include import ( - _ "github.com/elastic/beats/metricbeat/module/docker" + // Import packages that need to register themselves. _ "github.com/elastic/beats/metricbeat/module/docker/container" _ "github.com/elastic/beats/metricbeat/module/docker/cpu" _ "github.com/elastic/beats/metricbeat/module/docker/diskio" @@ -38,7 +32,6 @@ import ( _ "github.com/elastic/beats/metricbeat/module/docker/info" _ "github.com/elastic/beats/metricbeat/module/docker/memory" _ "github.com/elastic/beats/metricbeat/module/docker/network" - _ "github.com/elastic/beats/metricbeat/module/kubernetes" _ "github.com/elastic/beats/metricbeat/module/kubernetes/apiserver" _ "github.com/elastic/beats/metricbeat/module/kubernetes/container" _ "github.com/elastic/beats/metricbeat/module/kubernetes/controllermanager" @@ -56,6 +49,5 @@ import ( _ "github.com/elastic/beats/metricbeat/module/kubernetes/state_resourcequota" _ "github.com/elastic/beats/metricbeat/module/kubernetes/state_statefulset" _ "github.com/elastic/beats/metricbeat/module/kubernetes/system" - _ "github.com/elastic/beats/metricbeat/module/kubernetes/util" _ "github.com/elastic/beats/metricbeat/module/kubernetes/volume" ) diff --git a/metricbeat/magefile.go b/metricbeat/magefile.go index 2464e0a6a6a0..e1521193fe18 100644 --- a/metricbeat/magefile.go +++ b/metricbeat/magefile.go @@ -91,6 +91,12 @@ func Config() { mg.Deps(configYML, metricbeat.GenerateDirModulesD) } +// Imports generates an include/list_{suffix}.go file containing +// a import statement for each module and dataset. +func Imports() error { + return metricbeat.GenerateOSSMetricbeatModuleIncludeListGo() +} + func configYML() error { return devtools.Config(devtools.AllConfigTypes, metricbeat.OSSConfigFileParams(), ".") } diff --git a/metricbeat/scripts/generate_imports_helper.py b/metricbeat/scripts/generate_imports_helper.py deleted file mode 100644 index 3a1df4a35a04..000000000000 --- a/metricbeat/scripts/generate_imports_helper.py +++ /dev/null @@ -1,65 +0,0 @@ -from os.path import abspath, isdir, join -from os import listdir - -comment = """Package include imports all Module and MetricSet packages so that they register -their factories with the global registry. This package can be imported in the -main package to automatically register all of the standard supported Metricbeat -modules.""" - -modules_by_platform = [ - { - "file_suffix": "_docker", - "build_tags": "// +build linux darwin windows\n\n", - "modules": ["docker", "kubernetes"], - }, -] - - -def get_importable_lines(go_beat_path, import_line): - path = abspath("module") - - imports_by_module = [] - common_lines = [] - modules = [m for m in listdir(path) if isdir(join(path, m)) and m != "_meta"] - not_common_modules = [] - for m in modules_by_platform: - not_common_modules.extend(m["modules"]) - - for platform_info in modules_by_platform: - lines = [] - for module in modules: - module_import = import_line.format(beat_path=go_beat_path, module="module", name=module) - if module in not_common_modules: - lines = _collect_imports_from_module(path, module, module_import, go_beat_path, import_line, lines) - else: - common_lines = _collect_imports_from_module( - path, module, module_import, go_beat_path, import_line, common_lines) - - if lines is not None: - imports_by_module.append({ - "file_suffix": platform_info["file_suffix"], - "build_tags": platform_info["build_tags"], - "imported_lines": lines, - }) - - imports_by_module.append({ - "file_suffix": "_common", - "build_tags": "", - "imported_lines": sorted(common_lines), - }) - - return imports_by_module - - -def _collect_imports_from_module(path, module, module_import, go_beat_path, import_line, imported_lines): - imported_lines.append(module_import) - - module_path = join(path, module) - ignore = ["_meta", "vendor", "mtest"] - metricsets = [m for m in listdir(module_path) if isdir(join(module_path, m)) and m not in ignore] - for metricset in metricsets: - metricset_name = "{}/{}".format(module, metricset) - metricset_import = import_line.format(beat_path=go_beat_path, module="module", name=metricset_name) - imported_lines.append(metricset_import) - - return sorted(imported_lines) diff --git a/metricbeat/scripts/mage/fields.go b/metricbeat/scripts/mage/fields.go new file mode 100644 index 000000000000..df3441ecf861 --- /dev/null +++ b/metricbeat/scripts/mage/fields.go @@ -0,0 +1,52 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mage + +import ( + devtools "github.com/elastic/beats/dev-tools/mage" +) + +// GenerateOSSMetricbeatModuleIncludeListGo generates include/list_{suffix}.go files containing +// a import statement for each module and dataset. +func GenerateOSSMetricbeatModuleIncludeListGo() error { + // generate include/list_common.go + err := devtools.GenerateIncludeListGo( + devtools.IncludeListOptions{ + ImportDirs: nil, + ModuleDirs: []string{"module"}, + ModulesToExclude: []string{"module/docker", "module/kubernetes"}, + Outfile: "include/list_common.go", + BuildTags: "", + Pkg: "include"}) + if err != nil { + return err + } + // generate include/list_docker.go + err = devtools.GenerateIncludeListGo( + devtools.IncludeListOptions{ + ImportDirs: nil, + ModuleDirs: []string{"module/docker", "module/kubernetes"}, + ModulesToExclude: nil, + Outfile: "include/list_docker.go", + BuildTags: "\n// +build linux darwin windows\n", + Pkg: "include"}) + if err != nil { + return err + } + return nil +} diff --git a/packetbeat/magefile.go b/packetbeat/magefile.go index 252f4a8c2b60..9b68ee91a8ff 100644 --- a/packetbeat/magefile.go +++ b/packetbeat/magefile.go @@ -150,7 +150,10 @@ func Config() error { } func includeList() error { - return devtools.GenerateIncludeListGo([]string{"protos/*"}, nil) + options := devtools.DefaultIncludeListOptions() + options.ImportDirs = []string{"protos/*"} + options.ModuleDirs = nil + return devtools.GenerateIncludeListGo(options) } // Fields generates fields.yml and fields.go files for the Beat. diff --git a/script/generate_imports.py b/script/generate_imports.py deleted file mode 100644 index 055eda716c08..000000000000 --- a/script/generate_imports.py +++ /dev/null @@ -1,63 +0,0 @@ -import sys -from os import listdir, getcwd -from os.path import abspath, isdir, join, dirname, basename -from argparse import ArgumentParser - -sys.path.append(abspath("scripts")) -from generate_imports_helper import comment, get_importable_lines - - -import_line_format = "\t_ \"{beat_path}/{module}/{name}\"" -import_template = """// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -// Code generated by 'make imports' - DO NOT EDIT. - -{buildtags}/* -{comment} -*/ -package {package} - -import ( -{imports} -) -""" - - -def generate_and_write_to_file(outfile, go_beat_path): - outputs = get_importable_lines(go_beat_path, import_line_format) - - for output_data in outputs: - if len(output_data["imported_lines"]) == 0: - continue - imported_lines = "\n".join(output_data["imported_lines"]) - package = basename(dirname(outfile)) - list_go = import_template.format(package=package, - comment=comment, - buildtags=output_data["build_tags"], - imports=imported_lines) - with open(outfile.format(suffix=output_data["file_suffix"]), "w") as output: - output.write(list_go) - - -if __name__ == "__main__": - parser = ArgumentParser(description="Generate imports for Beats packages") - parser.add_argument("--out", default="include/list{suffix}.go") - parser.add_argument("beats_path") - args = parser.parse_args() - - generate_and_write_to_file(args.out, args.beats_path) diff --git a/x-pack/filebeat/magefile.go b/x-pack/filebeat/magefile.go index 2ccc0ff0b143..4aa1d8851ce0 100644 --- a/x-pack/filebeat/magefile.go +++ b/x-pack/filebeat/magefile.go @@ -134,7 +134,9 @@ func Update() { } func includeList() error { - return devtools.GenerateIncludeListGo([]string{"input/*", "processors/*"}, []string{"module"}) + options := devtools.DefaultIncludeListOptions() + options.ImportDirs = []string{"input/*", "processors/*"} + return devtools.GenerateIncludeListGo(options) } // IntegTest executes integration tests (it uses Docker to run the tests).