mage: optionally mount module cache for crossbuild (#16837) (#16947)

* mage: optionally mount module cache for crossbuild

Introduce another mage variable, CrossBuildMountModcache,
which defaults to false. When set to true, the host's
module cache ($GOPATH/pkg/mod) will be mounted into the
crossbuild Docker containers, read-only. To ensure the
cache is up-to-date, we run "go mod download" on the host
before starting the Docker containers.

Also, fix buildGoDaemon to stop assuming that tsg/go-daemon
is vendored. Instead, use "go list -m github.com/tsg/go-daemon"
to find the directory in either the vendor directory or
the module cache.

Rather than having arguments for ListModulePath, create separate
functions for listing in the module cache and vendor directory. Create
an unexported function in dev-tools/mage which calls the appropriate
one depending on the value of UseVendor. We now use this in
"findElasticBeatsDir".

Inline defaultConfigFileParams, otherwise the OSSBeatDir and
LibbeatDir calls end up calling ElasticBeatsDir at package init time,
before apm-server uas an opportunity to set UseVendor=false.

(cherry picked from commit bbf9d66)

dev-tools/mage/common.go

@@ -49,6 +49,8 @@ import (
 	"github.com/magefile/mage/sh"
 	"github.com/magefile/mage/target"
 	"github.com/pkg/errors"
+
+	"github.com/elastic/beats/v7/dev-tools/mage/gotool"
 )
 
 // Expand expands the given Go text/template string.
@@ -769,3 +771,13 @@ func binaryExtension(goos string) string {
 	}
 	return ""
 }
+
+// listModuleDir calls gotool.ListModuleVendorDir or
+// gotool.ListModuleCacheDir, depending on the value of
+// UseVendor.
+func listModuleDir(modpath string) (string, error) {
+	if UseVendor {
+		return gotool.ListModuleVendorDir(modpath)
+	}
+	return gotool.ListModuleCacheDir(modpath)
+}

dev-tools/mage/config.go

@@ -38,21 +38,6 @@ var (
 	shortTemplate     = filepath.Join("build", BeatName+".yml.tmpl")
 	referenceTemplate = filepath.Join("build", BeatName+".reference.yml.tmpl")
 	dockerTemplate    = filepath.Join("build", BeatName+".docker.yml.tmpl")
-
-	defaultConfigFileParams = ConfigFileParams{
-		ShortParts: []string{
-			OSSBeatDir("_meta/beat.yml"),
-			LibbeatDir("_meta/config.yml.tmpl"),
-		},
-		ReferenceParts: []string{
-			OSSBeatDir("_meta/beat.reference.yml"),
-			LibbeatDir("_meta/config.reference.yml.tmpl"),
-		},
-		DockerParts: []string{
-			OSSBeatDir("_meta/beat.docker.yml"),
-			LibbeatDir("_meta/config.docker.yml"),
-		},
-	}
 )
 
 // ConfigFileType is a bitset that indicates what types of config files to
@@ -97,7 +82,20 @@ func (c ConfigFileParams) Empty() bool {
 // host for the generated configs. Defaults to linux/amd64.
 func Config(types ConfigFileType, args ConfigFileParams, targetDir string) error {
 	if args.Empty() {
-		args = defaultConfigFileParams
+		args = ConfigFileParams{
+			ShortParts: []string{
+				OSSBeatDir("_meta/beat.yml"),
+				LibbeatDir("_meta/config.yml.tmpl"),
+			},
+			ReferenceParts: []string{
+				OSSBeatDir("_meta/beat.reference.yml"),
+				LibbeatDir("_meta/config.reference.yml.tmpl"),
+			},
+			DockerParts: []string{
+				OSSBeatDir("_meta/beat.docker.yml"),
+				LibbeatDir("_meta/config.docker.yml"),
+			},
+		}
 	}
 
 	if err := makeConfigTemplates(types, args); err != nil {

dev-tools/mage/crossbuild.go

@@ -19,6 +19,7 @@ package mage
 
 import (
 	"fmt"
+	"go/build"
 	"log"
 	"os"
 	"path/filepath"
@@ -31,6 +32,7 @@ import (
 	"github.com/magefile/mage/sh"
 	"github.com/pkg/errors"
 
+	"github.com/elastic/beats/v7/dev-tools/mage/gotool"
 	"github.com/elastic/beats/v7/libbeat/common/file"
 )
 
@@ -127,6 +129,12 @@ func CrossBuild(options ...CrossBuildOption) error {
 		return nil
 	}
 
+	if CrossBuildMountModcache {
+		// Make sure the module dependencies are downloaded on the host,
+		// as they will be mounted into the container read-only.
+		mg.Deps(func() error { return gotool.Mod.Download() })
+	}
+
 	// Build the magefile for Linux so we can run it inside the container.
 	mg.Deps(buildMage)
 
@@ -250,6 +258,11 @@ func (b GolangCrossBuilder) Build() error {
 	if UseVendor {
 		args = append(args, "--env", "GOFLAGS=-mod=vendor")
 	}
+	if CrossBuildMountModcache {
+		// Mount $GOPATH/pkg/mod into the container, read-only.
+		hostDir := filepath.Join(build.Default.GOPATH, "pkg", "mod")
+		args = append(args, "-v", hostDir+":/go/pkg/mod:ro")
+	}
 
 	args = append(args,
 		"--rm",

dev-tools/mage/godaemon.go

@@ -21,6 +21,7 @@ import (
 	"errors"
 	"log"
 	"os"
+	"path/filepath"
 )
 
 var (
@@ -42,8 +43,12 @@ func BuildGoDaemon() error {
 	}
 
 	// Test if binaries are up-to-date.
+	godaemonDir, err := listModuleDir("github.com/tsg/go-daemon")
+	if err != nil {
+		return err
+	}
+	input := filepath.Join(godaemonDir, "src", "god.c")
 	output := MustExpand("build/golang-crossbuild/god-{{.Platform.GOOS}}-{{.Platform.Arch}}")
-	input := MustExpand("{{ elastic_beats_dir }}/vendor/github.com/tsg/go-daemon/src/god.c")
 	if IsUpToDate(output, input) {
 		log.Println(">>> buildGoDaemon is up-to-date for", Platform.Name)
 		return nil

dev-tools/mage/gomod.go

@@ -75,7 +75,7 @@ func Vendor() error {
 
 	// copy packages which require the whole tree
 	for _, p := range copyAll {
-		path, err := gotool.ListModulePath(p.name)
+		path, err := gotool.ListModuleVendorDir(p.name)
 		if err != nil {
 			return err
 		}

dev-tools/mage/gotool/go.go

@@ -93,14 +93,31 @@ func ListTestFiles(pkg string) ([]string, error) {
 	return getLines(callGo(nil, "list", "-f", tmpl, pkg))
 }
 
-// ListModulePath returns the path to the module in the cache.
-func ListModulePath(pkg string) (string, error) {
-	const tmpl = `{{.Dir}}`
+// ListModuleCacheDir returns the module cache directory containing
+// the specified module. If the module does not exist in the cache,
+// an error will be returned.
+func ListModuleCacheDir(pkg string) (string, error) {
+	return listModuleDir(pkg, false)
+}
+
+// ListModuleVendorDir returns the vendor directory containing the
+// specified module. If the module has not been vendored, an error
+// will be returned.
+func ListModuleVendorDir(pkg string) (string, error) {
+	return listModuleDir(pkg, true)
+}
+
+func listModuleDir(pkg string, vendor bool) (string, error) {
 	env := map[string]string{
-		// make sure to look in the module cache
+		// Make sure GOFLAGS does not influence behaviour.
 		"GOFLAGS": "",
 	}
-	lines, err := getLines(callGo(env, "list", "-m", "-f", tmpl, pkg))
+	args := []string{"-m", "-f", "{{.Dir}}"}
+	if vendor {
+		args = append(args, "-mod=vendor")
+	}
+	args = append(args, pkg)
+	lines, err := getLines(callGo(env, "list", args...))
 	if err != nil {
 		return "", err
 	}

dev-tools/mage/gotool/modules.go

@@ -19,10 +19,11 @@ package gotool
 
 // Mod is the command go mod.
 var Mod = goMod{
-	Init:   modCommand{"init"}.run,
-	Tidy:   modCommand{"tidy"}.run,
-	Verify: modCommand{"verify"}.run,
-	Vendor: modCommand{"vendor"}.run,
+	Download: modCommand{"download"}.run,
+	Init:     modCommand{"init"}.run,
+	Tidy:     modCommand{"tidy"}.run,
+	Verify:   modCommand{"verify"}.run,
+	Vendor:   modCommand{"vendor"}.run,
 }
 
 type modCommand struct {
@@ -40,12 +41,16 @@ func (cmd modCommand) run(opts ...ArgOpt) error {
 }
 
 type goMod struct {
-	Init   modInit
-	Tidy   modTidy
-	Verify modVerify
-	Vendor modVendor
+	Download modDownload
+	Init     modInit
+	Tidy     modTidy
+	Verify   modVerify
+	Vendor   modVendor
 }
 
+// modDownload cleans the go.mod file
+type modDownload func(opts ...ArgOpt) error
+
 // modInit initializes a new go module in folder.
 type modInit func(opts ...ArgOpt) error
 

dev-tools/mage/settings.go

@@ -60,6 +60,10 @@ var (
 	TestCoverage = false
 	UseVendor    = true
 
+	// CrossBuildMountModcache, if true, mounts $GOPATH/pkg/mod into
+	// the crossbuild images at /go/pkg/mod, read-only.
+	CrossBuildMountModcache = false
+
 	BeatName        = EnvOr("BEAT_NAME", filepath.Base(CWD()))
 	BeatServiceName = EnvOr("BEAT_SERVICE_NAME", BeatName)
 	BeatIndexPrefix = EnvOr("BEAT_INDEX_PREFIX", BeatName)
@@ -280,8 +284,7 @@ func findElasticBeatsDir() (string, error) {
 	if repo.IsElasticBeats() {
 		return repo.RootDir, nil
 	}
-
-	return gotool.ListModulePath("github.com/elastic/beats/v7")
+	return listModuleDir("github.com/elastic/beats/v7")
 }
 
 var (

generator/common/beatgen/beatgen.go

@@ -101,6 +101,12 @@ func Generate() error {
 		return err
 	}
 
+	// Make sure the ElasticBeatsDir value is cached
+	// before changing directory below.
+	if _, err := devtools.ElasticBeatsDir(); err != nil {
+		return err
+	}
+
 	err = setup.GenNewBeat(cfg)
 	if err != nil {
 		return errors.Wrap(err, "error generating new beat")

generator/common/beatgen/setup/setup.go

@@ -112,7 +112,7 @@ func CopyVendor() error {
 		return err
 	}
 
-	path, err := gotool.ListModulePath("github.com/elastic/beats/v7")
+	path, err := gotool.ListModuleCacheDir("github.com/elastic/beats/v7")
 	if err != nil {
 		return err
 	}