Replace generate_imports python script with mage imports

dev-tools/cmd/module_include_list/module_include_list.go

@@ -50,18 +50,22 @@ Options:
 `[1:]
 
 var (
-	license    string
-	pkg        string
-	outFile    string
-	moduleDirs stringSliceFlag
-	importDirs stringSliceFlag
+	license           string
+	pkg               string
+	outFile           string
+	buildTags         string
+	moduleDirs        stringSliceFlag
+	moduleExcludeDirs stringSliceFlag
+	importDirs        stringSliceFlag
 )
 
 func init() {
 	flag.StringVar(&license, "license", "ASL2", "License header for generated file (ASL2 or Elastic).")
 	flag.StringVar(&pkg, "pkg", "include", "Package name.")
 	flag.StringVar(&outFile, "out", "include/list.go", "Output file.")
+	flag.StringVar(&buildTags, "buildTags", "", "Build Tags.")
 	flag.Var(&moduleDirs, "moduleDir", "Directory to search for modules to include")
+	flag.Var(&moduleExcludeDirs, "moduleExcludeDirs", "Directory to exclude from the list")
 	flag.Var(&importDirs, "import", "Directory to include")
 	flag.Usage = usageFlag
 }
@@ -138,9 +142,10 @@ func main() {
 	// Populate the template.
 	var buf bytes.Buffer
 	err = Template.Execute(&buf, Data{
-		License: license,
-		Package: pkg,
-		Imports: imports,
+		License:   license,
+		Package:   pkg,
+		BuildTags: buildTags,
+		Imports:   imports,
 	})
 	if err != nil {
 		log.Fatalf("Failed executing template: %v", err)
@@ -168,7 +173,7 @@ var Template = template.Must(template.New("normalizations").Funcs(map[string]int
 {{ .License | trim }}
 
 // Code generated by beats/dev-tools/cmd/module_include_list/module_include_list.go - DO NOT EDIT.
-
+{{ .BuildTags }}
 package {{ .Package }}
 
 import (
@@ -180,9 +185,10 @@ import (
 `[1:]))
 
 type Data struct {
-	License string
-	Package string
-	Imports []string
+	License   string
+	Package   string
+	BuildTags string
+	Imports   []string
 }
 
 //stringSliceFlag is a flag type that allows more than one value to be specified.
@@ -211,6 +217,16 @@ func findModuleAndDatasets() ([]string, error) {
 
 		for _, metaDir := range metaDirs {
 			// Strip off _meta.
+			skipDir := false
+			for _, excludeModule := range moduleExcludeDirs {
+				if strings.Contains(metaDir, excludeModule) {
+					skipDir = true
+					break
+				}
+			}
+			if skipDir {
+				continue
+			}
 			dirs = append(dirs, filepath.Dir(metaDir))
 		}
 	}

dev-tools/mage/fields.go

@@ -37,6 +37,26 @@ const (
 	FieldsAllYML = "build/fields/fields.all.yml"
 )
 
+type IncludeListOptions struct {
+	ImportDirs       []string
+	ModuleDirs       []string
+	ModulesToExclude []string
+	Outfile          string
+	BuildTags        string
+	Pkg              string
+}
+
+func DefaultIncludeListOptions() IncludeListOptions {
+	return IncludeListOptions{
+		ImportDirs:       nil,
+		ModuleDirs:       []string{"module"},
+		ModulesToExclude: nil,
+		Outfile:          "include/list.go",
+		BuildTags:        "",
+		Pkg:              "include",
+	}
+}
+
 // FieldsBuilder is the interface projects to implement for building field data.
 type FieldsBuilder interface {
 	// Generate all fields.go files.
@@ -143,13 +163,13 @@ func GenerateModuleFieldsGo(moduleDir string) error {
 // GenerateModuleIncludeListGo generates an include/list.go file containing
 // a import statement for each module and dataset.
 func GenerateModuleIncludeListGo() error {
-	return GenerateIncludeListGo(nil, []string{"module"})
+	return GenerateIncludeListGo(DefaultIncludeListOptions())
 }
 
 // GenerateIncludeListGo generates an include/list.go file containing imports
 // for the packages that match the paths (or globs) in importDirs (optional)
 // and moduleDirs (optional).
-func GenerateIncludeListGo(importDirs []string, moduleDirs []string) error {
+func GenerateIncludeListGo(options IncludeListOptions) error {
 	const moduleIncludeListCmdPath = "dev-tools/cmd/module_include_list/module_include_list.go"
 
 	beatsDir, err := ElasticBeatsDir()
@@ -160,22 +180,29 @@ func GenerateIncludeListGo(importDirs []string, moduleDirs []string) error {
 	includeListCmd := sh.RunCmd("go", "run",
 		filepath.Join(beatsDir, moduleIncludeListCmdPath),
 		"-license", toLibbeatLicenseName(BeatLicense),
+		"-out", options.Outfile, "-buildTags", options.BuildTags,
+		"-pkg", options.Pkg,
 	)
 
 	var args []string
-	for _, dir := range importDirs {
+	for _, dir := range options.ImportDirs {
 		if !filepath.IsAbs(dir) {
 			dir = CWD(dir)
 		}
 		args = append(args, "-import", dir)
 	}
-	for _, dir := range moduleDirs {
+	for _, dir := range options.ModuleDirs {
 		if !filepath.IsAbs(dir) {
 			dir = CWD(dir)
 		}
 		args = append(args, "-moduleDir", dir)
 	}
-
+	for _, dir := range options.ModulesToExclude {
+		if !filepath.IsAbs(dir) {
+			dir = CWD(dir)
+		}
+		args = append(args, "-moduleExcludeDirs", dir)
+	}
 	return includeListCmd(args...)
 }
 

filebeat/magefile.go

@@ -112,7 +112,9 @@ func configYML() error {
 
 // includeList generates include/list.go with imports for inputs.
 func includeList() error {
-	return devtools.GenerateIncludeListGo([]string{"input/*"}, []string{"module"})
+	options := devtools.DefaultIncludeListOptions()
+	options.ImportDirs = []string{"input/*"}
+	return devtools.GenerateIncludeListGo(options)
 }
 
 // Fields generates fields.yml and fields.go files for the Beat.

generator/metricbeat/{beat}/magefile.go

@@ -104,6 +104,12 @@ func Update() error {
 	return update.Update()
 }
 
+// Imports generates an include/list.go file containing
+// a import statement for each module and dataset.
+func Imports() error {
+	return devtools.GenerateModuleIncludeListGo()
+}
+
 // Test runs all available tests
 func Test() {
 	mg.Deps(unittest.GoUnitTest)

heartbeat/Makefile

@@ -12,9 +12,9 @@ collect: imports kibana
 
 # Generate imports for all monitors
 .PHONY: imports
-imports: python-env
+imports:
 	@mkdir -p include
-	@${PYTHON_ENV}/bin/python ${ES_BEATS}/script/generate_imports.py --out monitors/defaults/default.go ${BEAT_PATH}
+	mage imports
 
 # Collects all module dashboards
 .PHONY: kibana

heartbeat/magefile.go

@@ -110,6 +110,16 @@ func Fields() error {
 	return devtools.GenerateFieldsYAML("monitors/active")
 }
 
+// Imports generates an include/list.go file containing
+// a import statement for each module and dataset.
+func Imports() error {
+	options := devtools.DefaultIncludeListOptions()
+	options.ModuleDirs = []string{"monitors"}
+	options.Outfile = "monitors/defaults/default.go"
+	options.Pkg = "defaults"
+	return devtools.GenerateIncludeListGo(options)
+}
+
 // GoTestUnit executes the Go unit tests.
 // Use TEST_COVERAGE=true to enable code coverage profiling.
 // Use RACE_DETECTOR=true to enable the race detector.

metricbeat/Makefile

@@ -41,9 +41,9 @@ configs: python-env
 
 # Generates imports for all modules and metricsets
 .PHONY: imports
-imports: python-env
+imports:
 	@mkdir -p include
-	@${PYTHON_ENV}/bin/python ${ES_BEATS}/script/generate_imports.py ${BEAT_PATH}
+	mage imports
 
 # Runs all collection steps and updates afterwards
 .PHONY: collect

metricbeat/magefile.go

@@ -91,6 +91,12 @@ func Config() {
 	mg.Deps(configYML, metricbeat.GenerateDirModulesD)
 }
 
+// Imports generates an include/list_{suffix}.go file containing
+// a import statement for each module and dataset.
+func Imports() error {
+	return metricbeat.GenerateOSSMetricbeatModuleIncludeListGo()
+}
+
 func configYML() error {
 	return devtools.Config(devtools.AllConfigTypes, metricbeat.OSSConfigFileParams(), ".")
 }

metricbeat/scripts/mage/fields.go

@@ -0,0 +1,50 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package mage
+
+import (
+	devtools "github.com/elastic/beats/dev-tools/mage"
+)
+
+// GenerateOSSMetricbeatModuleIncludeListGo generates include/list_{suffix}.go files containing
+// a import statement for each module and dataset.
+func GenerateOSSMetricbeatModuleIncludeListGo() error {
+	err := devtools.GenerateIncludeListGo(
+		devtools.IncludeListOptions{
+			nil,
+			[]string{"module"},
+			[]string{"module/docker", "module/kubernetes"},
+			"include/list_common.go",
+			"",
+			"include"})
+	if err != nil {
+		return err
+	}
+	err = devtools.GenerateIncludeListGo(
+		devtools.IncludeListOptions{
+			nil,
+			[]string{"module/docker", "module/kubernetes"},
+			nil,
+			"include/list_docker.go",
+			"\n// +build linux darwin windows\n",
+			"include"})
+	if err != nil {
+		return err
+	}
+	return nil
+}

packetbeat/magefile.go

@@ -150,7 +150,9 @@ func Config() error {
 }
 
 func includeList() error {
-	return devtools.GenerateIncludeListGo([]string{"protos/*"}, nil)
+	options := devtools.DefaultIncludeListOptions()
+	options.ImportDirs = []string{"protos/*"}
+	return devtools.GenerateIncludeListGo(options)
 }
 
 // Fields generates fields.yml and fields.go files for the Beat.

x-pack/filebeat/magefile.go

@@ -134,7 +134,9 @@ func Update() {
 }
 
 func includeList() error {
-	return devtools.GenerateIncludeListGo([]string{"input/*", "processors/*"}, []string{"module"})
+	options := devtools.DefaultIncludeListOptions()
+	options.ImportDirs = []string{"input/*", "processors/*"}
+	return devtools.GenerateIncludeListGo(options)
 }
 
 // IntegTest executes integration tests (it uses Docker to run the tests).