Replace generate_imports python script with mage imports (#14498)

dev-tools/cmd/module_include_list/module_include_list.go

@@ -50,18 +50,22 @@ Options:
 `[1:]
 
 var (
-	license    string
-	pkg        string
-	outFile    string
-	moduleDirs stringSliceFlag
-	importDirs stringSliceFlag
+	license           string
+	pkg               string
+	outFile           string
+	buildTags         string
+	moduleDirs        stringSliceFlag
+	moduleExcludeDirs stringSliceFlag
+	importDirs        stringSliceFlag
 )
 
 func init() {
 	flag.StringVar(&license, "license", "ASL2", "License header for generated file (ASL2 or Elastic).")
 	flag.StringVar(&pkg, "pkg", "include", "Package name.")
 	flag.StringVar(&outFile, "out", "include/list.go", "Output file.")
+	flag.StringVar(&buildTags, "buildTags", "", "Build Tags.")
 	flag.Var(&moduleDirs, "moduleDir", "Directory to search for modules to include")
+	flag.Var(&moduleExcludeDirs, "moduleExcludeDirs", "Directory to exclude from the list")
 	flag.Var(&importDirs, "import", "Directory to include")
 	flag.Usage = usageFlag
 }
@@ -138,9 +142,10 @@ func main() {
 	// Populate the template.
 	var buf bytes.Buffer
 	err = Template.Execute(&buf, Data{
-		License: license,
-		Package: pkg,
-		Imports: imports,
+		License:   license,
+		Package:   pkg,
+		BuildTags: buildTags,
+		Imports:   imports,
 	})
 	if err != nil {
 		log.Fatalf("Failed executing template: %v", err)
@@ -168,7 +173,7 @@ var Template = template.Must(template.New("normalizations").Funcs(map[string]int
 {{ .License | trim }}
 
 // Code generated by beats/dev-tools/cmd/module_include_list/module_include_list.go - DO NOT EDIT.
-
+{{ .BuildTags }}
 package {{ .Package }}
 
 import (
@@ -180,9 +185,10 @@ import (
 `[1:]))
 
 type Data struct {
-	License string
-	Package string
-	Imports []string
+	License   string
+	Package   string
+	BuildTags string
+	Imports   []string
 }
 
 //stringSliceFlag is a flag type that allows more than one value to be specified.
@@ -211,6 +217,16 @@ func findModuleAndDatasets() ([]string, error) {
 
 		for _, metaDir := range metaDirs {
 			// Strip off _meta.
+			skipDir := false
+			for _, excludeModule := range moduleExcludeDirs {
+				if strings.Contains(metaDir, excludeModule) {
+					skipDir = true
+					break
+				}
+			}
+			if skipDir {
+				continue
+			}
 			dirs = append(dirs, filepath.Dir(metaDir))
 		}
 	}

dev-tools/mage/fields.go

@@ -37,6 +37,28 @@ const (
 	FieldsAllYML = "build/fields/fields.all.yml"
 )
 
+// IncludeListOptions stores the options for IncludeList generation
+type IncludeListOptions struct {
+	ImportDirs       []string
+	ModuleDirs       []string
+	ModulesToExclude []string
+	Outfile          string
+	BuildTags        string
+	Pkg              string
+}
+
+// DefaultIncludeListOptions initializes IncludeListOptions struct with default values
+func DefaultIncludeListOptions() IncludeListOptions {
+	return IncludeListOptions{
+		ImportDirs:       nil,
+		ModuleDirs:       []string{"module"},
+		ModulesToExclude: nil,
+		Outfile:          "include/list.go",
+		BuildTags:        "",
+		Pkg:              "include",
+	}
+}
+
 // FieldsBuilder is the interface projects to implement for building field data.
 type FieldsBuilder interface {
 	// Generate all fields.go files.
@@ -143,13 +165,13 @@ func GenerateModuleFieldsGo(moduleDir string) error {
 // GenerateModuleIncludeListGo generates an include/list.go file containing
 // a import statement for each module and dataset.
 func GenerateModuleIncludeListGo() error {
-	return GenerateIncludeListGo(nil, []string{"module"})
+	return GenerateIncludeListGo(DefaultIncludeListOptions())
 }
 
 // GenerateIncludeListGo generates an include/list.go file containing imports
 // for the packages that match the paths (or globs) in importDirs (optional)
 // and moduleDirs (optional).
-func GenerateIncludeListGo(importDirs []string, moduleDirs []string) error {
+func GenerateIncludeListGo(options IncludeListOptions) error {
 	const moduleIncludeListCmdPath = "dev-tools/cmd/module_include_list/module_include_list.go"
 
 	beatsDir, err := ElasticBeatsDir()
@@ -160,22 +182,29 @@ func GenerateIncludeListGo(importDirs []string, moduleDirs []string) error {
 	includeListCmd := sh.RunCmd("go", "run",
 		filepath.Join(beatsDir, moduleIncludeListCmdPath),
 		"-license", toLibbeatLicenseName(BeatLicense),
+		"-out", options.Outfile, "-buildTags", options.BuildTags,
+		"-pkg", options.Pkg,
 	)
 
 	var args []string
-	for _, dir := range importDirs {
+	for _, dir := range options.ImportDirs {
 		if !filepath.IsAbs(dir) {
 			dir = CWD(dir)
 		}
 		args = append(args, "-import", dir)
 	}
-	for _, dir := range moduleDirs {
+	for _, dir := range options.ModuleDirs {
 		if !filepath.IsAbs(dir) {
 			dir = CWD(dir)
 		}
 		args = append(args, "-moduleDir", dir)
 	}
-
+	for _, dir := range options.ModulesToExclude {
+		if !filepath.IsAbs(dir) {
+			dir = CWD(dir)
+		}
+		args = append(args, "-moduleExcludeDirs", dir)
+	}
 	return includeListCmd(args...)
 }
 

filebeat/magefile.go

@@ -112,7 +112,9 @@ func configYML() error {
 
 // includeList generates include/list.go with imports for inputs.
 func includeList() error {
-	return devtools.GenerateIncludeListGo([]string{"input/*"}, []string{"module"})
+	options := devtools.DefaultIncludeListOptions()
+	options.ImportDirs = []string{"input/*"}
+	return devtools.GenerateIncludeListGo(options)
 }
 
 // Fields generates fields.yml and fields.go files for the Beat.

generator/common/beatgen/beatgen.go

@@ -129,8 +129,7 @@ func VendorUpdate() error {
 	}
 
 	devtools.SetElasticBeatsDir(getAbsoluteBeatsPath())
-	mg.SerialDeps(setup.CopyVendor, setup.LinkImportsHelper)
-	return nil
+	return setup.CopyVendor()
 }
 
 // returns a "compleated" config object with everything we need

generator/common/beatgen/setup/setup.go

@@ -48,17 +48,7 @@ func RunSetup() error {
 	if err != nil {
 		return errors.Wrapf(err, "error copying pkg to %s", vendorPath)
 	}
-	return LinkImportsHelper()
-}
-
-// LinkImportsHelper links generate_imports_helper.py
-func LinkImportsHelper() error {
-	vendorPath := "./vendor/github.com/"
-	pwd, err := os.Getwd()
-	if err != nil {
-		return errors.Wrap(err, "error gettting current directory")
-	}
-	return sh.Run("ln", "-sf", filepath.Join(pwd, vendorPath, "elastic/beats/metricbeat/scripts/generate_imports_helper.py"), filepath.Join(pwd, vendorPath, "elastic/beats/script/generate_imports_helper.py"))
+	return nil
 }
 
 // CopyVendor copies a new version of beats into the vendor directory of PWD

generator/metricbeat/{beat}/magefile.go

@@ -104,6 +104,12 @@ func Update() error {
 	return update.Update()
 }
 
+// Imports generates an include/list.go file containing
+// a import statement for each module and dataset.
+func Imports() error {
+	return devtools.GenerateModuleIncludeListGo()
+}
+
 // Test runs all available tests
 func Test() {
 	mg.Deps(unittest.GoUnitTest)

heartbeat/Makefile

@@ -12,9 +12,9 @@ collect: imports kibana
 
 # Generate imports for all monitors
 .PHONY: imports
-imports: python-env
+imports:
 	@mkdir -p include
-	@${PYTHON_ENV}/bin/python ${ES_BEATS}/script/generate_imports.py --out monitors/defaults/default.go ${BEAT_PATH}
+	mage imports
 
 # Collects all module dashboards
 .PHONY: kibana

heartbeat/magefile.go

@@ -110,6 +110,16 @@ func Fields() error {
 	return devtools.GenerateFieldsYAML("monitors/active")
 }
 
+// Imports generates an include/list.go file containing
+// a import statement for each module and dataset.
+func Imports() error {
+	options := devtools.DefaultIncludeListOptions()
+	options.ModuleDirs = []string{"monitors"}
+	options.Outfile = "monitors/defaults/default.go"
+	options.Pkg = "defaults"
+	return devtools.GenerateIncludeListGo(options)
+}
+
 // GoTestUnit executes the Go unit tests.
 // Use TEST_COVERAGE=true to enable code coverage profiling.
 // Use RACE_DETECTOR=true to enable the race detector.

metricbeat/Makefile

@@ -41,9 +41,9 @@ configs: python-env
 
 # Generates imports for all modules and metricsets
 .PHONY: imports
-imports: python-env
+imports:
 	@mkdir -p include
-	@${PYTHON_ENV}/bin/python ${ES_BEATS}/script/generate_imports.py ${BEAT_PATH}
+	mage imports
 
 # Runs all collection steps and updates afterwards
 .PHONY: collect

metricbeat/magefile.go

@@ -91,6 +91,12 @@ func Config() {
 	mg.Deps(configYML, metricbeat.GenerateDirModulesD)
 }
 
+// Imports generates an include/list_{suffix}.go file containing
+// a import statement for each module and dataset.
+func Imports() error {
+	return metricbeat.GenerateOSSMetricbeatModuleIncludeListGo()
+}
+
 func configYML() error {
 	return devtools.Config(devtools.AllConfigTypes, metricbeat.OSSConfigFileParams(), ".")
 }