From 6cdcc3b65148e086644f3235a61470f44b6abdce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Thu, 19 Mar 2020 09:29:45 +0100 Subject: [PATCH 1/4] Copy missing go version to build --- .travis.yml | 21 ++++++++++----------- generator/common/beatgen/setup/setup.go | 1 + 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index 96490883a6ab..ddeb794dc2c3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -191,17 +191,16 @@ jobs: stage: test # Generators - # https://github.com/elastic/beats/issues/16951 - #- os: linux - # before_install: .ci/scripts/travis_has_changes.sh generator metricbeat libbeat || travis_terminate 0 - # env: TARGETS="-C generator/_templates/metricbeat test test-package" - # go: $TRAVIS_GO_VERSION - # stage: test - #- os: linux - # before_install: .ci/scripts/travis_has_changes.sh generator libbeat || travis_terminate 0 - # env: TARGETS="-C generator/_templates/beat test test-package" - # go: $TRAVIS_GO_VERSION - # stage: test + - os: linux + before_install: .ci/scripts/travis_has_changes.sh generator metricbeat libbeat || travis_terminate 0 + env: TARGETS="-C generator/_templates/metricbeat test test-package" + go: $TRAVIS_GO_VERSION + stage: test + - os: linux + before_install: .ci/scripts/travis_has_changes.sh generator libbeat || travis_terminate 0 + env: TARGETS="-C generator/_templates/beat test test-package" + go: $TRAVIS_GO_VERSION + stage: test - os: osx before_install: .ci/scripts/travis_has_changes.sh generator metricbeat libbeat || travis_terminate 0 diff --git a/generator/common/beatgen/setup/setup.go b/generator/common/beatgen/setup/setup.go index b372660971d7..4392e9122d7f 100644 --- a/generator/common/beatgen/setup/setup.go +++ b/generator/common/beatgen/setup/setup.go @@ -36,6 +36,7 @@ var ( "licenses", "metricbeat", "script", + ".go-version", } ) From f29b8f199bbe6b563fadda9f61f3fb76346f8c19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Thu, 19 Mar 2020 12:33:52 +0100 Subject: [PATCH 2/4] add files needed for packageing --- dev-tools/mage/gomod.go | 62 +++++++++++-------- .../_templates/beat/{beat}/tools/tools.go | 1 + .../metricbeat/{beat}/tools/tools.go | 1 + generator/common/beatgen/setup/setup.go | 44 ++++++------- 4 files changed, 59 insertions(+), 49 deletions(-) diff --git a/dev-tools/mage/gomod.go b/dev-tools/mage/gomod.go index ae136665b1b0..86b8d6d3587b 100644 --- a/dev-tools/mage/gomod.go +++ b/dev-tools/mage/gomod.go @@ -24,30 +24,30 @@ import ( "github.com/elastic/beats/v7/dev-tools/mage/gotool" ) -// copyModule contains a module name and the list of files or directories +// CopyModule contains a module name and the list of files or directories // to copy recursively. -type copyModule struct { - name string - filesToCopy []string +type CopyModule struct { + Name string + FilesToCopy []string } var ( - copyAll = []copyModule{ - copyModule{ - name: "github.com/godror/godror", - filesToCopy: []string{ - "odpi", + copyAll = []CopyModule{ + CopyModule{ + Name: "github.com/tsg/go-daemon", + FilesToCopy: []string{ + "src", }, }, - copyModule{ - name: "github.com/tsg/go-daemon", - filesToCopy: []string{ - "src", + CopyModule{ + Name: "github.com/godror/godror", + FilesToCopy: []string{ + "odpi", }, }, } filesToRemove = []string{ - filepath.Join("vendor", "github.com", "yuin", "gopher-lua", "parse", "Makefile"), + filepath.Join("github.com", "yuin", "gopher-lua", "parse", "Makefile"), } ) @@ -74,18 +74,34 @@ func Vendor() error { if err != nil { return err } + vendorFolder := filepath.Join(repo.RootDir, "vendor") + err = CopyFilesToVendor(vendorFolder, copyAll) + if err != nil { + return err + } + + for _, p := range filesToRemove { + p = filepath.Join(vendorFolder, p) + err = os.RemoveAll(p) + if err != nil { + return err + } + } + return nil +} - // copy packages which require the whole tree - for _, p := range copyAll { - path, err := gotool.ListModuleCacheDir(p.name) +// CopyFilesToVendor copies packages which require the whole tree +func CopyFilesToVendor(vendorFolder string, modulesToCopy []CopyModule) error { + for _, p := range modulesToCopy { + path, err := gotool.ListModuleCacheDir(p.Name) if err != nil { return err } - for _, f := range p.filesToCopy { + for _, f := range p.FilesToCopy { from := filepath.Join(path, f) - to := filepath.Join(vendorFolder, p.name, f) + to := filepath.Join(vendorFolder, p.Name, f) copyTask := &CopyTask{Source: from, Dest: to, DirMode: os.ModeDir | 0750} err = copyTask.Execute() if err != nil { @@ -93,13 +109,5 @@ func Vendor() error { } } } - - for _, p := range filesToRemove { - p = filepath.Join(repo.RootDir, p) - err = os.RemoveAll(p) - if err != nil { - return err - } - } return nil } diff --git a/generator/_templates/beat/{beat}/tools/tools.go b/generator/_templates/beat/{beat}/tools/tools.go index d816e58647fd..edd121e211b9 100644 --- a/generator/_templates/beat/{beat}/tools/tools.go +++ b/generator/_templates/beat/{beat}/tools/tools.go @@ -6,6 +6,7 @@ package tools import ( _ "github.com/pierrre/gotestcover" + _ "github.com/tsg/go-daemon" _ "golang.org/x/tools/cmd/goimports" _ "github.com/mitchellh/gox" diff --git a/generator/_templates/metricbeat/{beat}/tools/tools.go b/generator/_templates/metricbeat/{beat}/tools/tools.go index d816e58647fd..edd121e211b9 100644 --- a/generator/_templates/metricbeat/{beat}/tools/tools.go +++ b/generator/_templates/metricbeat/{beat}/tools/tools.go @@ -6,6 +6,7 @@ package tools import ( _ "github.com/pierrre/gotestcover" + _ "github.com/tsg/go-daemon" _ "golang.org/x/tools/cmd/goimports" _ "github.com/mitchellh/gox" diff --git a/generator/common/beatgen/setup/setup.go b/generator/common/beatgen/setup/setup.go index 4392e9122d7f..5585d83131a9 100644 --- a/generator/common/beatgen/setup/setup.go +++ b/generator/common/beatgen/setup/setup.go @@ -29,17 +29,6 @@ import ( "github.com/elastic/beats/v7/dev-tools/mage/gotool" ) -var ( - makefileDeps = []string{ - "dev-tools", - "libbeat", - "licenses", - "metricbeat", - "script", - ".go-version", - } -) - func InitModule() error { err := gotool.Mod.Init() if err != nil { @@ -113,21 +102,32 @@ func CopyVendor() error { return err } - path, err := gotool.ListModuleCacheDir("github.com/elastic/beats/v7") + err = devtools.CopyFilesToVendor( + "./vendor", + []devtools.CopyModule{ + devtools.CopyModule{ + Name: "github.com/elastic/beats/v7", + FilesToCopy: []string{ + "dev-tools", + "libbeat", + "licenses", + "metricbeat", + "script", + ".go-version", + }, + }, + devtools.CopyModule{ + Name: "github.com/tsg/go-daemon", + FilesToCopy: []string{ + "src", + }, + }, + }, + ) if err != nil { return err } - vendorPath := "./vendor/github.com/elastic/beats/v7" - for _, d := range makefileDeps { - from := filepath.Join(path, d) - to := filepath.Join(vendorPath, d) - copyTask := &devtools.CopyTask{Source: from, Dest: to, Mode: 0640, DirMode: os.ModeDir | 0750} - err = copyTask.Execute() - if err != nil { - return err - } - } return nil } From 314e1b6b508978285f1ebe6f84461d78bb4611b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Thu, 19 Mar 2020 14:29:23 +0100 Subject: [PATCH 3/4] change access --- dev-tools/mage/gomod.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-tools/mage/gomod.go b/dev-tools/mage/gomod.go index 86b8d6d3587b..779ef5c996e4 100644 --- a/dev-tools/mage/gomod.go +++ b/dev-tools/mage/gomod.go @@ -102,7 +102,7 @@ func CopyFilesToVendor(vendorFolder string, modulesToCopy []CopyModule) error { for _, f := range p.FilesToCopy { from := filepath.Join(path, f) to := filepath.Join(vendorFolder, p.Name, f) - copyTask := &CopyTask{Source: from, Dest: to, DirMode: os.ModeDir | 0750} + copyTask := &CopyTask{Source: from, Dest: to, Mode: 0640, DirMode: os.ModeDir | 0750} err = copyTask.Execute() if err != nil { return err From 39c702dd6cdbb485ef45db3d9074f12d23bb6611 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Thu, 19 Mar 2020 16:13:43 +0100 Subject: [PATCH 4/4] add error wrapping --- dev-tools/mage/gomod.go | 20 +++++++++++--------- generator/common/beatgen/setup/setup.go | 4 ++-- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/dev-tools/mage/gomod.go b/dev-tools/mage/gomod.go index 779ef5c996e4..dcff94563aeb 100644 --- a/dev-tools/mage/gomod.go +++ b/dev-tools/mage/gomod.go @@ -21,6 +21,8 @@ import ( "os" "path/filepath" + "github.com/pkg/errors" + "github.com/elastic/beats/v7/dev-tools/mage/gotool" ) @@ -57,35 +59,35 @@ func Vendor() error { err := mod.Tidy() if err != nil { - return err + return errors.Wrap(err, "error while running go mod tidy") } err = mod.Vendor() if err != nil { - return err + return errors.Wrap(err, "error while running go mod vendor") } err = mod.Verify() if err != nil { - return err + return errors.Wrap(err, "error while running go mod verify") } repo, err := GetProjectRepoInfo() if err != nil { - return err + return errors.Wrap(err, "error while getting repository information") } vendorFolder := filepath.Join(repo.RootDir, "vendor") err = CopyFilesToVendor(vendorFolder, copyAll) if err != nil { - return err + return errors.Wrap(err, "error copying required files") } for _, p := range filesToRemove { p = filepath.Join(vendorFolder, p) err = os.RemoveAll(p) if err != nil { - return err + return errors.Wrapf(err, "error while removing file: %s", p) } } return nil @@ -96,16 +98,16 @@ func CopyFilesToVendor(vendorFolder string, modulesToCopy []CopyModule) error { for _, p := range modulesToCopy { path, err := gotool.ListModuleCacheDir(p.Name) if err != nil { - return err + return errors.Wrapf(err, "error while looking up cached dir of module: %s", p.Name) } for _, f := range p.FilesToCopy { from := filepath.Join(path, f) to := filepath.Join(vendorFolder, p.Name, f) - copyTask := &CopyTask{Source: from, Dest: to, Mode: 0640, DirMode: os.ModeDir | 0750} + copyTask := &CopyTask{Source: from, Dest: to, Mode: 0600, DirMode: os.ModeDir | 0750} err = copyTask.Execute() if err != nil { - return err + return errors.Wrapf(err, "error while copying file from %s to %s", from, to) } } } diff --git a/generator/common/beatgen/setup/setup.go b/generator/common/beatgen/setup/setup.go index 5585d83131a9..42098c40f4de 100644 --- a/generator/common/beatgen/setup/setup.go +++ b/generator/common/beatgen/setup/setup.go @@ -99,7 +99,7 @@ func copyReplacedModules() error { func CopyVendor() error { err := gotool.Mod.Vendor() if err != nil { - return err + return errors.Wrapf(err, "error while running go mod vendor") } err = devtools.CopyFilesToVendor( @@ -125,7 +125,7 @@ func CopyVendor() error { }, ) if err != nil { - return err + return errors.Wrapf(err, "error while copying required files to vendor") } return nil