From efcd2bdbd90a444d84eb8096da49c9c248a43233 Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Wed, 6 Nov 2024 14:39:27 -0500 Subject: [PATCH] internal/packagestest: fork go/packages/packagestest Also, update all imports outside of that package. Updates golang/go#70229 Change-Id: I8b08f892ec86d560c0406319c2954eb9912d78ad Reviewed-on: https://go-review.googlesource.com/c/tools/+/625920 LUCI-TryBot-Result: Go LUCI Reviewed-by: Robert Findley --- cmd/bundle/main_test.go | 2 +- cmd/godoc/godoc_test.go | 2 +- go/analysis/unitchecker/unitchecker_test.go | 2 +- go/buildutil/allpackages_test.go | 2 +- go/buildutil/util_test.go | 2 +- go/packages/overlay_test.go | 2 +- go/packages/packages_test.go | 2 +- go/ssa/ssautil/load_test.go | 2 +- internal/imports/fix_test.go | 2 +- internal/packagestest/expect.go | 468 ++++++++++++ internal/packagestest/expect_test.go | 71 ++ internal/packagestest/export.go | 666 ++++++++++++++++++ internal/packagestest/export_test.go | 234 ++++++ internal/packagestest/gopath.go | 77 ++ internal/packagestest/gopath_test.go | 28 + internal/packagestest/modules.go | 223 ++++++ internal/packagestest/modules_test.go | 32 + .../one/modules/example.com/extra/help.go | 1 + .../testdata/groups/one/primarymod/main.go | 1 + .../modules/example.com/extra/geez/help.go | 1 + .../modules/example.com/extra/v2/geez/help.go | 1 + .../two/modules/example.com/extra/v2/me.go | 1 + .../two/modules/example.com/extra/yo.go | 1 + .../two/modules/example.com/tempmod/main.go | 1 + .../modules/example.com/what@v1.0.0/main.go | 1 + .../modules/example.com/what@v1.1.0/main.go | 1 + .../groups/two/primarymod/expect/yo.go | 3 + .../groups/two/primarymod/expect/yo_test.go | 10 + .../testdata/groups/two/primarymod/main.go | 1 + internal/packagestest/testdata/test.go | 24 + internal/packagestest/testdata/test_test.go | 3 + internal/packagestest/testdata/x_test.go | 3 + refactor/importgraph/graph_test.go | 2 +- 33 files changed, 1862 insertions(+), 10 deletions(-) create mode 100644 internal/packagestest/expect.go create mode 100644 internal/packagestest/expect_test.go create mode 100644 internal/packagestest/export.go create mode 100644 internal/packagestest/export_test.go create mode 100644 internal/packagestest/gopath.go create mode 100644 internal/packagestest/gopath_test.go create mode 100644 internal/packagestest/modules.go create mode 100644 internal/packagestest/modules_test.go create mode 100644 internal/packagestest/testdata/groups/one/modules/example.com/extra/help.go create mode 100644 internal/packagestest/testdata/groups/one/primarymod/main.go create mode 100644 internal/packagestest/testdata/groups/two/modules/example.com/extra/geez/help.go create mode 100644 internal/packagestest/testdata/groups/two/modules/example.com/extra/v2/geez/help.go create mode 100644 internal/packagestest/testdata/groups/two/modules/example.com/extra/v2/me.go create mode 100644 internal/packagestest/testdata/groups/two/modules/example.com/extra/yo.go create mode 100644 internal/packagestest/testdata/groups/two/modules/example.com/tempmod/main.go create mode 100644 internal/packagestest/testdata/groups/two/modules/example.com/what@v1.0.0/main.go create mode 100644 internal/packagestest/testdata/groups/two/modules/example.com/what@v1.1.0/main.go create mode 100644 internal/packagestest/testdata/groups/two/primarymod/expect/yo.go create mode 100644 internal/packagestest/testdata/groups/two/primarymod/expect/yo_test.go create mode 100644 internal/packagestest/testdata/groups/two/primarymod/main.go create mode 100644 internal/packagestest/testdata/test.go create mode 100644 internal/packagestest/testdata/test_test.go create mode 100644 internal/packagestest/testdata/x_test.go diff --git a/cmd/bundle/main_test.go b/cmd/bundle/main_test.go index 4ee8521a074..0dee2afb0b2 100644 --- a/cmd/bundle/main_test.go +++ b/cmd/bundle/main_test.go @@ -11,7 +11,7 @@ import ( "runtime" "testing" - "golang.org/x/tools/go/packages/packagestest" + "golang.org/x/tools/internal/packagestest" ) func TestBundle(t *testing.T) { packagestest.TestAll(t, testBundle) } diff --git a/cmd/godoc/godoc_test.go b/cmd/godoc/godoc_test.go index 2269ace3f7c..94159445a54 100644 --- a/cmd/godoc/godoc_test.go +++ b/cmd/godoc/godoc_test.go @@ -21,7 +21,7 @@ import ( "testing" "time" - "golang.org/x/tools/go/packages/packagestest" + "golang.org/x/tools/internal/packagestest" "golang.org/x/tools/internal/testenv" ) diff --git a/go/analysis/unitchecker/unitchecker_test.go b/go/analysis/unitchecker/unitchecker_test.go index 54d8fa81851..1801b49cfe8 100644 --- a/go/analysis/unitchecker/unitchecker_test.go +++ b/go/analysis/unitchecker/unitchecker_test.go @@ -17,7 +17,7 @@ import ( "golang.org/x/tools/go/analysis/passes/findcall" "golang.org/x/tools/go/analysis/passes/printf" "golang.org/x/tools/go/analysis/unitchecker" - "golang.org/x/tools/go/packages/packagestest" + "golang.org/x/tools/internal/packagestest" ) func TestMain(m *testing.M) { diff --git a/go/buildutil/allpackages_test.go b/go/buildutil/allpackages_test.go index 1aa194d868e..6af86771104 100644 --- a/go/buildutil/allpackages_test.go +++ b/go/buildutil/allpackages_test.go @@ -17,7 +17,7 @@ import ( "testing" "golang.org/x/tools/go/buildutil" - "golang.org/x/tools/go/packages/packagestest" + "golang.org/x/tools/internal/packagestest" ) func TestAllPackages(t *testing.T) { diff --git a/go/buildutil/util_test.go b/go/buildutil/util_test.go index 6c507579a38..534828d969b 100644 --- a/go/buildutil/util_test.go +++ b/go/buildutil/util_test.go @@ -13,7 +13,7 @@ import ( "testing" "golang.org/x/tools/go/buildutil" - "golang.org/x/tools/go/packages/packagestest" + "golang.org/x/tools/internal/packagestest" ) func TestContainingPackage(t *testing.T) { diff --git a/go/packages/overlay_test.go b/go/packages/overlay_test.go index 5760b7774b3..9edd0d646ed 100644 --- a/go/packages/overlay_test.go +++ b/go/packages/overlay_test.go @@ -14,7 +14,7 @@ import ( "testing" "golang.org/x/tools/go/packages" - "golang.org/x/tools/go/packages/packagestest" + "golang.org/x/tools/internal/packagestest" "golang.org/x/tools/internal/testenv" ) diff --git a/go/packages/packages_test.go b/go/packages/packages_test.go index be457893655..939f2df2da4 100644 --- a/go/packages/packages_test.go +++ b/go/packages/packages_test.go @@ -27,8 +27,8 @@ import ( "time" "golang.org/x/tools/go/packages" - "golang.org/x/tools/go/packages/packagestest" "golang.org/x/tools/internal/packagesinternal" + "golang.org/x/tools/internal/packagestest" "golang.org/x/tools/internal/testenv" "golang.org/x/tools/internal/testfiles" ) diff --git a/go/ssa/ssautil/load_test.go b/go/ssa/ssautil/load_test.go index efa2ba40a8b..10375a3227f 100644 --- a/go/ssa/ssautil/load_test.go +++ b/go/ssa/ssautil/load_test.go @@ -17,9 +17,9 @@ import ( "testing" "golang.org/x/tools/go/packages" - "golang.org/x/tools/go/packages/packagestest" "golang.org/x/tools/go/ssa" "golang.org/x/tools/go/ssa/ssautil" + "golang.org/x/tools/internal/packagestest" "golang.org/x/tools/internal/testenv" ) diff --git a/internal/imports/fix_test.go b/internal/imports/fix_test.go index e6e96ac2813..5409db0217f 100644 --- a/internal/imports/fix_test.go +++ b/internal/imports/fix_test.go @@ -20,8 +20,8 @@ import ( "sync/atomic" "testing" - "golang.org/x/tools/go/packages/packagestest" "golang.org/x/tools/internal/gocommand" + "golang.org/x/tools/internal/packagestest" "golang.org/x/tools/internal/stdlib" ) diff --git a/internal/packagestest/expect.go b/internal/packagestest/expect.go new file mode 100644 index 00000000000..053d8e8a9db --- /dev/null +++ b/internal/packagestest/expect.go @@ -0,0 +1,468 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packagestest + +import ( + "fmt" + "go/token" + "os" + "path/filepath" + "reflect" + "regexp" + "strings" + + "golang.org/x/tools/go/packages" + "golang.org/x/tools/internal/expect" +) + +const ( + markMethod = "mark" + eofIdentifier = "EOF" +) + +// Expect invokes the supplied methods for all expectation notes found in +// the exported source files. +// +// All exported go source files are parsed to collect the expectation +// notes. +// See the documentation for expect.Parse for how the notes are collected +// and parsed. +// +// The methods are supplied as a map of name to function, and those functions +// will be matched against the expectations by name. +// Notes with no matching function will be skipped, and functions with no +// matching notes will not be invoked. +// If there are no registered markers yet, a special pass will be run first +// which adds any markers declared with @mark(Name, pattern) or @name. These +// call the Mark method to add the marker to the global set. +// You can register the "mark" method to override these in your own call to +// Expect. The bound Mark function is usable directly in your method map, so +// +// exported.Expect(map[string]interface{}{"mark": exported.Mark}) +// +// replicates the built in behavior. +// +// # Method invocation +// +// When invoking a method the expressions in the parameter list need to be +// converted to values to be passed to the method. +// There are a very limited set of types the arguments are allowed to be. +// +// expect.Note : passed the Note instance being evaluated. +// string : can be supplied either a string literal or an identifier. +// int : can only be supplied an integer literal. +// *regexp.Regexp : can only be supplied a regular expression literal +// token.Pos : has a file position calculated as described below. +// token.Position : has a file position calculated as described below. +// expect.Range: has a start and end position as described below. +// interface{} : will be passed any value +// +// # Position calculation +// +// There is some extra handling when a parameter is being coerced into a +// token.Pos, token.Position or Range type argument. +// +// If the parameter is an identifier, it will be treated as the name of an +// marker to look up (as if markers were global variables). +// +// If it is a string or regular expression, then it will be passed to +// expect.MatchBefore to look up a match in the line at which it was declared. +// +// It is safe to call this repeatedly with different method sets, but it is +// not safe to call it concurrently. +func (e *Exported) Expect(methods map[string]interface{}) error { + if err := e.getNotes(); err != nil { + return err + } + if err := e.getMarkers(); err != nil { + return err + } + var err error + ms := make(map[string]method, len(methods)) + for name, f := range methods { + mi := method{f: reflect.ValueOf(f)} + mi.converters = make([]converter, mi.f.Type().NumIn()) + for i := 0; i < len(mi.converters); i++ { + mi.converters[i], err = e.buildConverter(mi.f.Type().In(i)) + if err != nil { + return fmt.Errorf("invalid method %v: %v", name, err) + } + } + ms[name] = mi + } + for _, n := range e.notes { + if n.Args == nil { + // simple identifier form, convert to a call to mark + n = &expect.Note{ + Pos: n.Pos, + Name: markMethod, + Args: []interface{}{n.Name, n.Name}, + } + } + mi, ok := ms[n.Name] + if !ok { + continue + } + params := make([]reflect.Value, len(mi.converters)) + args := n.Args + for i, convert := range mi.converters { + params[i], args, err = convert(n, args) + if err != nil { + return fmt.Errorf("%v: %v", e.ExpectFileSet.Position(n.Pos), err) + } + } + if len(args) > 0 { + return fmt.Errorf("%v: unwanted args got %+v extra", e.ExpectFileSet.Position(n.Pos), args) + } + //TODO: catch the error returned from the method + mi.f.Call(params) + } + return nil +} + +// A Range represents an interval within a source file in go/token notation. +type Range struct { + TokFile *token.File // non-nil + Start, End token.Pos // both valid and within range of TokFile +} + +// Mark adds a new marker to the known set. +func (e *Exported) Mark(name string, r Range) { + if e.markers == nil { + e.markers = make(map[string]Range) + } + e.markers[name] = r +} + +func (e *Exported) getNotes() error { + if e.notes != nil { + return nil + } + notes := []*expect.Note{} + var dirs []string + for _, module := range e.written { + for _, filename := range module { + dirs = append(dirs, filepath.Dir(filename)) + } + } + for filename := range e.Config.Overlay { + dirs = append(dirs, filepath.Dir(filename)) + } + pkgs, err := packages.Load(e.Config, dirs...) + if err != nil { + return fmt.Errorf("unable to load packages for directories %s: %v", dirs, err) + } + seen := make(map[token.Position]struct{}) + for _, pkg := range pkgs { + for _, filename := range pkg.GoFiles { + content, err := e.FileContents(filename) + if err != nil { + return err + } + l, err := expect.Parse(e.ExpectFileSet, filename, content) + if err != nil { + return fmt.Errorf("failed to extract expectations: %v", err) + } + for _, note := range l { + pos := e.ExpectFileSet.Position(note.Pos) + if _, ok := seen[pos]; ok { + continue + } + notes = append(notes, note) + seen[pos] = struct{}{} + } + } + } + if _, ok := e.written[e.primary]; !ok { + e.notes = notes + return nil + } + // Check go.mod markers regardless of mode, we need to do this so that our marker count + // matches the counts in the summary.txt.golden file for the test directory. + if gomod, found := e.written[e.primary]["go.mod"]; found { + // If we are in Modules mode, then we need to check the contents of the go.mod.temp. + if e.Exporter == Modules { + gomod += ".temp" + } + l, err := goModMarkers(e, gomod) + if err != nil { + return fmt.Errorf("failed to extract expectations for go.mod: %v", err) + } + notes = append(notes, l...) + } + e.notes = notes + return nil +} + +func goModMarkers(e *Exported, gomod string) ([]*expect.Note, error) { + if _, err := os.Stat(gomod); os.IsNotExist(err) { + // If there is no go.mod file, we want to be able to continue. + return nil, nil + } + content, err := e.FileContents(gomod) + if err != nil { + return nil, err + } + if e.Exporter == GOPATH { + return expect.Parse(e.ExpectFileSet, gomod, content) + } + gomod = strings.TrimSuffix(gomod, ".temp") + // If we are in Modules mode, copy the original contents file back into go.mod + if err := os.WriteFile(gomod, content, 0644); err != nil { + return nil, nil + } + return expect.Parse(e.ExpectFileSet, gomod, content) +} + +func (e *Exported) getMarkers() error { + if e.markers != nil { + return nil + } + // set markers early so that we don't call getMarkers again from Expect + e.markers = make(map[string]Range) + return e.Expect(map[string]interface{}{ + markMethod: e.Mark, + }) +} + +var ( + noteType = reflect.TypeOf((*expect.Note)(nil)) + identifierType = reflect.TypeOf(expect.Identifier("")) + posType = reflect.TypeOf(token.Pos(0)) + positionType = reflect.TypeOf(token.Position{}) + rangeType = reflect.TypeOf(Range{}) + fsetType = reflect.TypeOf((*token.FileSet)(nil)) + regexType = reflect.TypeOf((*regexp.Regexp)(nil)) + exportedType = reflect.TypeOf((*Exported)(nil)) +) + +// converter converts from a marker's argument parsed from the comment to +// reflect values passed to the method during Invoke. +// It takes the args remaining, and returns the args it did not consume. +// This allows a converter to consume 0 args for well known types, or multiple +// args for compound types. +type converter func(*expect.Note, []interface{}) (reflect.Value, []interface{}, error) + +// method is used to track information about Invoke methods that is expensive to +// calculate so that we can work it out once rather than per marker. +type method struct { + f reflect.Value // the reflect value of the passed in method + converters []converter // the parameter converters for the method +} + +// buildConverter works out what function should be used to go from an ast expressions to a reflect +// value of the type expected by a method. +// It is called when only the target type is know, it returns converters that are flexible across +// all supported expression types for that target type. +func (e *Exported) buildConverter(pt reflect.Type) (converter, error) { + switch { + case pt == noteType: + return func(n *expect.Note, args []interface{}) (reflect.Value, []interface{}, error) { + return reflect.ValueOf(n), args, nil + }, nil + case pt == fsetType: + return func(n *expect.Note, args []interface{}) (reflect.Value, []interface{}, error) { + return reflect.ValueOf(e.ExpectFileSet), args, nil + }, nil + case pt == exportedType: + return func(n *expect.Note, args []interface{}) (reflect.Value, []interface{}, error) { + return reflect.ValueOf(e), args, nil + }, nil + case pt == posType: + return func(n *expect.Note, args []interface{}) (reflect.Value, []interface{}, error) { + r, remains, err := e.rangeConverter(n, args) + if err != nil { + return reflect.Value{}, nil, err + } + return reflect.ValueOf(r.Start), remains, nil + }, nil + case pt == positionType: + return func(n *expect.Note, args []interface{}) (reflect.Value, []interface{}, error) { + r, remains, err := e.rangeConverter(n, args) + if err != nil { + return reflect.Value{}, nil, err + } + return reflect.ValueOf(e.ExpectFileSet.Position(r.Start)), remains, nil + }, nil + case pt == rangeType: + return func(n *expect.Note, args []interface{}) (reflect.Value, []interface{}, error) { + r, remains, err := e.rangeConverter(n, args) + if err != nil { + return reflect.Value{}, nil, err + } + return reflect.ValueOf(r), remains, nil + }, nil + case pt == identifierType: + return func(n *expect.Note, args []interface{}) (reflect.Value, []interface{}, error) { + if len(args) < 1 { + return reflect.Value{}, nil, fmt.Errorf("missing argument") + } + arg := args[0] + args = args[1:] + switch arg := arg.(type) { + case expect.Identifier: + return reflect.ValueOf(arg), args, nil + default: + return reflect.Value{}, nil, fmt.Errorf("cannot convert %v to string", arg) + } + }, nil + + case pt == regexType: + return func(n *expect.Note, args []interface{}) (reflect.Value, []interface{}, error) { + if len(args) < 1 { + return reflect.Value{}, nil, fmt.Errorf("missing argument") + } + arg := args[0] + args = args[1:] + if _, ok := arg.(*regexp.Regexp); !ok { + return reflect.Value{}, nil, fmt.Errorf("cannot convert %v to *regexp.Regexp", arg) + } + return reflect.ValueOf(arg), args, nil + }, nil + + case pt.Kind() == reflect.String: + return func(n *expect.Note, args []interface{}) (reflect.Value, []interface{}, error) { + if len(args) < 1 { + return reflect.Value{}, nil, fmt.Errorf("missing argument") + } + arg := args[0] + args = args[1:] + switch arg := arg.(type) { + case expect.Identifier: + return reflect.ValueOf(string(arg)), args, nil + case string: + return reflect.ValueOf(arg), args, nil + default: + return reflect.Value{}, nil, fmt.Errorf("cannot convert %v to string", arg) + } + }, nil + case pt.Kind() == reflect.Int64: + return func(n *expect.Note, args []interface{}) (reflect.Value, []interface{}, error) { + if len(args) < 1 { + return reflect.Value{}, nil, fmt.Errorf("missing argument") + } + arg := args[0] + args = args[1:] + switch arg := arg.(type) { + case int64: + return reflect.ValueOf(arg), args, nil + default: + return reflect.Value{}, nil, fmt.Errorf("cannot convert %v to int", arg) + } + }, nil + case pt.Kind() == reflect.Bool: + return func(n *expect.Note, args []interface{}) (reflect.Value, []interface{}, error) { + if len(args) < 1 { + return reflect.Value{}, nil, fmt.Errorf("missing argument") + } + arg := args[0] + args = args[1:] + b, ok := arg.(bool) + if !ok { + return reflect.Value{}, nil, fmt.Errorf("cannot convert %v to bool", arg) + } + return reflect.ValueOf(b), args, nil + }, nil + case pt.Kind() == reflect.Slice: + return func(n *expect.Note, args []interface{}) (reflect.Value, []interface{}, error) { + converter, err := e.buildConverter(pt.Elem()) + if err != nil { + return reflect.Value{}, nil, err + } + result := reflect.MakeSlice(reflect.SliceOf(pt.Elem()), 0, len(args)) + for range args { + value, remains, err := converter(n, args) + if err != nil { + return reflect.Value{}, nil, err + } + result = reflect.Append(result, value) + args = remains + } + return result, args, nil + }, nil + default: + if pt.Kind() == reflect.Interface && pt.NumMethod() == 0 { + return func(n *expect.Note, args []interface{}) (reflect.Value, []interface{}, error) { + if len(args) < 1 { + return reflect.Value{}, nil, fmt.Errorf("missing argument") + } + return reflect.ValueOf(args[0]), args[1:], nil + }, nil + } + return nil, fmt.Errorf("param has unexpected type %v (kind %v)", pt, pt.Kind()) + } +} + +func (e *Exported) rangeConverter(n *expect.Note, args []interface{}) (Range, []interface{}, error) { + tokFile := e.ExpectFileSet.File(n.Pos) + if len(args) < 1 { + return Range{}, nil, fmt.Errorf("missing argument") + } + arg := args[0] + args = args[1:] + switch arg := arg.(type) { + case expect.Identifier: + // handle the special identifiers + switch arg { + case eofIdentifier: + // end of file identifier + eof := tokFile.Pos(tokFile.Size()) + return newRange(tokFile, eof, eof), args, nil + default: + // look up an marker by name + mark, ok := e.markers[string(arg)] + if !ok { + return Range{}, nil, fmt.Errorf("cannot find marker %v", arg) + } + return mark, args, nil + } + case string: + start, end, err := expect.MatchBefore(e.ExpectFileSet, e.FileContents, n.Pos, arg) + if err != nil { + return Range{}, nil, err + } + if !start.IsValid() { + return Range{}, nil, fmt.Errorf("%v: pattern %s did not match", e.ExpectFileSet.Position(n.Pos), arg) + } + return newRange(tokFile, start, end), args, nil + case *regexp.Regexp: + start, end, err := expect.MatchBefore(e.ExpectFileSet, e.FileContents, n.Pos, arg) + if err != nil { + return Range{}, nil, err + } + if !start.IsValid() { + return Range{}, nil, fmt.Errorf("%v: pattern %s did not match", e.ExpectFileSet.Position(n.Pos), arg) + } + return newRange(tokFile, start, end), args, nil + default: + return Range{}, nil, fmt.Errorf("cannot convert %v to pos", arg) + } +} + +// newRange creates a new Range from a token.File and two valid positions within it. +func newRange(file *token.File, start, end token.Pos) Range { + fileBase := file.Base() + fileEnd := fileBase + file.Size() + if !start.IsValid() { + panic("invalid start token.Pos") + } + if !end.IsValid() { + panic("invalid end token.Pos") + } + if int(start) < fileBase || int(start) > fileEnd { + panic(fmt.Sprintf("invalid start: %d not in [%d, %d]", start, fileBase, fileEnd)) + } + if int(end) < fileBase || int(end) > fileEnd { + panic(fmt.Sprintf("invalid end: %d not in [%d, %d]", end, fileBase, fileEnd)) + } + if start > end { + panic("invalid start: greater than end") + } + return Range{ + TokFile: file, + Start: start, + End: end, + } +} diff --git a/internal/packagestest/expect_test.go b/internal/packagestest/expect_test.go new file mode 100644 index 00000000000..d155f5fe9e2 --- /dev/null +++ b/internal/packagestest/expect_test.go @@ -0,0 +1,71 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packagestest_test + +import ( + "go/token" + "testing" + + "golang.org/x/tools/internal/expect" + "golang.org/x/tools/internal/packagestest" +) + +func TestExpect(t *testing.T) { + exported := packagestest.Export(t, packagestest.GOPATH, []packagestest.Module{{ + Name: "golang.org/fake", + Files: packagestest.MustCopyFileTree("testdata"), + }}) + defer exported.Cleanup() + checkCount := 0 + if err := exported.Expect(map[string]interface{}{ + "check": func(src, target token.Position) { + checkCount++ + }, + "boolArg": func(n *expect.Note, yes, no bool) { + if !yes { + t.Errorf("Expected boolArg first param to be true") + } + if no { + t.Errorf("Expected boolArg second param to be false") + } + }, + "intArg": func(n *expect.Note, i int64) { + if i != 42 { + t.Errorf("Expected intarg to be 42") + } + }, + "stringArg": func(n *expect.Note, name expect.Identifier, value string) { + if string(name) != value { + t.Errorf("Got string arg %v expected %v", value, name) + } + }, + "directNote": func(n *expect.Note) {}, + "range": func(r packagestest.Range) { + if r.Start == token.NoPos || r.Start == 0 { + t.Errorf("Range had no valid starting position") + } + if r.End == token.NoPos || r.End == 0 { + t.Errorf("Range had no valid ending position") + } else if r.End <= r.Start { + t.Errorf("Range ending was not greater than start") + } + }, + "checkEOF": func(n *expect.Note, p token.Pos) { + if p <= n.Pos { + t.Errorf("EOF was before the checkEOF note") + } + }, + }); err != nil { + t.Fatal(err) + } + // We expect to have walked the @check annotations in all .go files, + // including _test.go files (XTest or otherwise). But to have walked the + // non-_test.go files only once. Hence wantCheck = 3 (testdata/test.go) + 1 + // (testdata/test_test.go) + 1 (testdata/x_test.go) + wantCheck := 7 + if wantCheck != checkCount { + t.Fatalf("Expected @check count of %v; got %v", wantCheck, checkCount) + } +} diff --git a/internal/packagestest/export.go b/internal/packagestest/export.go new file mode 100644 index 00000000000..f8d10718c09 --- /dev/null +++ b/internal/packagestest/export.go @@ -0,0 +1,666 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +Package packagestest creates temporary projects on disk for testing go tools on. + +By changing the exporter used, you can create projects for multiple build +systems from the same description, and run the same tests on them in many +cases. + +# Example + +As an example of packagestest use, consider the following test that runs +the 'go list' command on the specified modules: + + // TestGoList exercises the 'go list' command in module mode and in GOPATH mode. + func TestGoList(t *testing.T) { packagestest.TestAll(t, testGoList) } + func testGoList(t *testing.T, x packagestest.Exporter) { + e := packagestest.Export(t, x, []packagestest.Module{ + { + Name: "gopher.example/repoa", + Files: map[string]interface{}{ + "a/a.go": "package a", + }, + }, + { + Name: "gopher.example/repob", + Files: map[string]interface{}{ + "b/b.go": "package b", + }, + }, + }) + defer e.Cleanup() + + cmd := exec.Command("go", "list", "gopher.example/...") + cmd.Dir = e.Config.Dir + cmd.Env = e.Config.Env + out, err := cmd.Output() + if err != nil { + t.Fatal(err) + } + t.Logf("'go list gopher.example/...' with %s mode layout:\n%s", x.Name(), out) + } + +TestGoList uses TestAll to exercise the 'go list' command with all +exporters known to packagestest. Currently, packagestest includes +exporters that produce module mode layouts and GOPATH mode layouts. +Running the test with verbose output will print: + + === RUN TestGoList + === RUN TestGoList/GOPATH + === RUN TestGoList/Modules + --- PASS: TestGoList (0.21s) + --- PASS: TestGoList/GOPATH (0.03s) + main_test.go:36: 'go list gopher.example/...' with GOPATH mode layout: + gopher.example/repoa/a + gopher.example/repob/b + --- PASS: TestGoList/Modules (0.18s) + main_test.go:36: 'go list gopher.example/...' with Modules mode layout: + gopher.example/repoa/a + gopher.example/repob/b +*/ +package packagestest + +import ( + "errors" + "flag" + "fmt" + "go/token" + "io" + "log" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "golang.org/x/tools/go/packages" + "golang.org/x/tools/internal/expect" + "golang.org/x/tools/internal/testenv" +) + +var ( + skipCleanup = flag.Bool("skip-cleanup", false, "Do not delete the temporary export folders") // for debugging +) + +// ErrUnsupported indicates an error due to an operation not supported on the +// current platform. +var ErrUnsupported = errors.New("operation is not supported") + +// Module is a representation of a go module. +type Module struct { + // Name is the base name of the module as it would be in the go.mod file. + Name string + // Files is the set of source files for all packages that make up the module. + // The keys are the file fragment that follows the module name, the value can + // be a string or byte slice, in which case it is the contents of the + // file, otherwise it must be a Writer function. + Files map[string]interface{} + + // Overlay is the set of source file overlays for the module. + // The keys are the file fragment as in the Files configuration. + // The values are the in memory overlay content for the file. + Overlay map[string][]byte +} + +// A Writer is a function that writes out a test file. +// It is provided the name of the file to write, and may return an error if it +// cannot write the file. +// These are used as the content of the Files map in a Module. +type Writer func(filename string) error + +// Exported is returned by the Export function to report the structure that was produced on disk. +type Exported struct { + // Config is a correctly configured packages.Config ready to be passed to packages.Load. + // Exactly what it will contain varies depending on the Exporter being used. + Config *packages.Config + + // Modules is the module description that was used to produce this exported data set. + Modules []Module + + ExpectFileSet *token.FileSet // The file set used when parsing expectations + + Exporter Exporter // the exporter used + temp string // the temporary directory that was exported to + primary string // the first non GOROOT module that was exported + written map[string]map[string]string // the full set of exported files + notes []*expect.Note // The list of expectations extracted from go source files + markers map[string]Range // The set of markers extracted from go source files +} + +// Exporter implementations are responsible for converting from the generic description of some +// test data to a driver specific file layout. +type Exporter interface { + // Name reports the name of the exporter, used in logging and sub-test generation. + Name() string + // Filename reports the system filename for test data source file. + // It is given the base directory, the module the file is part of and the filename fragment to + // work from. + Filename(exported *Exported, module, fragment string) string + // Finalize is called once all files have been written to write any extra data needed and modify + // the Config to match. It is handed the full list of modules that were encountered while writing + // files. + Finalize(exported *Exported) error +} + +// All is the list of known exporters. +// This is used by TestAll to run tests with all the exporters. +var All = []Exporter{GOPATH, Modules} + +// TestAll invokes the testing function once for each exporter registered in +// the All global. +// Each exporter will be run as a sub-test named after the exporter being used. +func TestAll(t *testing.T, f func(*testing.T, Exporter)) { + t.Helper() + for _, e := range All { + e := e // in case f calls t.Parallel + t.Run(e.Name(), func(t *testing.T) { + t.Helper() + f(t, e) + }) + } +} + +// BenchmarkAll invokes the testing function once for each exporter registered in +// the All global. +// Each exporter will be run as a sub-test named after the exporter being used. +func BenchmarkAll(b *testing.B, f func(*testing.B, Exporter)) { + b.Helper() + for _, e := range All { + e := e // in case f calls t.Parallel + b.Run(e.Name(), func(b *testing.B) { + b.Helper() + f(b, e) + }) + } +} + +// Export is called to write out a test directory from within a test function. +// It takes the exporter and the build system agnostic module descriptions, and +// uses them to build a temporary directory. +// It returns an Exported with the results of the export. +// The Exported.Config is prepared for loading from the exported data. +// You must invoke Exported.Cleanup on the returned value to clean up. +// The file deletion in the cleanup can be skipped by setting the skip-cleanup +// flag when invoking the test, allowing the temporary directory to be left for +// debugging tests. +// +// If the Writer for any file within any module returns an error equivalent to +// ErrUnspported, Export skips the test. +func Export(t testing.TB, exporter Exporter, modules []Module) *Exported { + t.Helper() + if exporter == Modules { + testenv.NeedsTool(t, "go") + } + + dirname := strings.Replace(t.Name(), "/", "_", -1) + dirname = strings.Replace(dirname, "#", "_", -1) // duplicate subtests get a #NNN suffix. + temp, err := os.MkdirTemp("", dirname) + if err != nil { + t.Fatal(err) + } + exported := &Exported{ + Config: &packages.Config{ + Dir: temp, + Env: append(os.Environ(), "GOPACKAGESDRIVER=off", "GOROOT="), // Clear GOROOT to work around #32849. + Overlay: make(map[string][]byte), + Tests: true, + Mode: packages.LoadImports, + }, + Modules: modules, + Exporter: exporter, + temp: temp, + primary: modules[0].Name, + written: map[string]map[string]string{}, + ExpectFileSet: token.NewFileSet(), + } + if testing.Verbose() { + exported.Config.Logf = t.Logf + } + defer func() { + if t.Failed() || t.Skipped() { + exported.Cleanup() + } + }() + for _, module := range modules { + // Create all parent directories before individual files. If any file is a + // symlink to a directory, that directory must exist before the symlink is + // created or else it may be created with the wrong type on Windows. + // (See https://golang.org/issue/39183.) + for fragment := range module.Files { + fullpath := exporter.Filename(exported, module.Name, filepath.FromSlash(fragment)) + if err := os.MkdirAll(filepath.Dir(fullpath), 0755); err != nil { + t.Fatal(err) + } + } + + for fragment, value := range module.Files { + fullpath := exporter.Filename(exported, module.Name, filepath.FromSlash(fragment)) + written, ok := exported.written[module.Name] + if !ok { + written = map[string]string{} + exported.written[module.Name] = written + } + written[fragment] = fullpath + switch value := value.(type) { + case Writer: + if err := value(fullpath); err != nil { + if errors.Is(err, ErrUnsupported) { + t.Skip(err) + } + t.Fatal(err) + } + case string: + if err := os.WriteFile(fullpath, []byte(value), 0644); err != nil { + t.Fatal(err) + } + default: + t.Fatalf("Invalid type %T in files, must be string or Writer", value) + } + } + for fragment, value := range module.Overlay { + fullpath := exporter.Filename(exported, module.Name, filepath.FromSlash(fragment)) + exported.Config.Overlay[fullpath] = value + } + } + if err := exporter.Finalize(exported); err != nil { + t.Fatal(err) + } + testenv.NeedsGoPackagesEnv(t, exported.Config.Env) + return exported +} + +// Script returns a Writer that writes out contents to the file and sets the +// executable bit on the created file. +// It is intended for source files that are shell scripts. +func Script(contents string) Writer { + return func(filename string) error { + return os.WriteFile(filename, []byte(contents), 0755) + } +} + +// Link returns a Writer that creates a hard link from the specified source to +// the required file. +// This is used to link testdata files into the generated testing tree. +// +// If hard links to source are not supported on the destination filesystem, the +// returned Writer returns an error for which errors.Is(_, ErrUnsupported) +// returns true. +func Link(source string) Writer { + return func(filename string) error { + linkErr := os.Link(source, filename) + + if linkErr != nil && !builderMustSupportLinks() { + // Probe to figure out whether Link failed because the Link operation + // isn't supported. + if stat, err := openAndStat(source); err == nil { + if err := createEmpty(filename, stat.Mode()); err == nil { + // Successfully opened the source and created the destination, + // but the result is empty and not a hard-link. + return &os.PathError{Op: "Link", Path: filename, Err: ErrUnsupported} + } + } + } + + return linkErr + } +} + +// Symlink returns a Writer that creates a symlink from the specified source to the +// required file. +// This is used to link testdata files into the generated testing tree. +// +// If symlinks to source are not supported on the destination filesystem, the +// returned Writer returns an error for which errors.Is(_, ErrUnsupported) +// returns true. +func Symlink(source string) Writer { + if !strings.HasPrefix(source, ".") { + if absSource, err := filepath.Abs(source); err == nil { + if _, err := os.Stat(source); !os.IsNotExist(err) { + source = absSource + } + } + } + return func(filename string) error { + symlinkErr := os.Symlink(source, filename) + + if symlinkErr != nil && !builderMustSupportLinks() { + // Probe to figure out whether Symlink failed because the Symlink + // operation isn't supported. + fullSource := source + if !filepath.IsAbs(source) { + // Compute the target path relative to the parent of filename, not the + // current working directory. + fullSource = filepath.Join(filename, "..", source) + } + stat, err := openAndStat(fullSource) + mode := os.ModePerm + if err == nil { + mode = stat.Mode() + } else if !errors.Is(err, os.ErrNotExist) { + // We couldn't open the source, but it might exist. We don't expect to be + // able to portably create a symlink to a file we can't see. + return symlinkErr + } + + if err := createEmpty(filename, mode|0644); err == nil { + // Successfully opened the source (or verified that it does not exist) and + // created the destination, but we couldn't create it as a symlink. + // Probably the OS just doesn't support symlinks in this context. + return &os.PathError{Op: "Symlink", Path: filename, Err: ErrUnsupported} + } + } + + return symlinkErr + } +} + +// builderMustSupportLinks reports whether we are running on a Go builder +// that is known to support hard and symbolic links. +func builderMustSupportLinks() bool { + if os.Getenv("GO_BUILDER_NAME") == "" { + // Any OS can be configured to mount an exotic filesystem. + // Don't make assumptions about what users are running. + return false + } + + switch runtime.GOOS { + case "windows", "plan9": + // Some versions of Windows and all versions of plan9 do not support + // symlinks by default. + return false + + default: + // All other platforms should support symlinks by default, and our builders + // should not do anything unusual that would violate that. + return true + } +} + +// openAndStat attempts to open source for reading. +func openAndStat(source string) (os.FileInfo, error) { + src, err := os.Open(source) + if err != nil { + return nil, err + } + stat, err := src.Stat() + src.Close() + if err != nil { + return nil, err + } + return stat, nil +} + +// createEmpty creates an empty file or directory (depending on mode) +// at dst, with the same permissions as mode. +func createEmpty(dst string, mode os.FileMode) error { + if mode.IsDir() { + return os.Mkdir(dst, mode.Perm()) + } + + f, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_EXCL, mode.Perm()) + if err != nil { + return err + } + if err := f.Close(); err != nil { + os.Remove(dst) // best-effort + return err + } + + return nil +} + +// Copy returns a Writer that copies a file from the specified source to the +// required file. +// This is used to copy testdata files into the generated testing tree. +func Copy(source string) Writer { + return func(filename string) error { + stat, err := os.Stat(source) + if err != nil { + return err + } + if !stat.Mode().IsRegular() { + // cannot copy non-regular files (e.g., directories, + // symlinks, devices, etc.) + return fmt.Errorf("cannot copy non regular file %s", source) + } + return copyFile(filename, source, stat.Mode().Perm()) + } +} + +func copyFile(dest, source string, perm os.FileMode) error { + src, err := os.Open(source) + if err != nil { + return err + } + defer src.Close() + + dst, err := os.OpenFile(dest, os.O_WRONLY|os.O_CREATE|os.O_EXCL, perm) + if err != nil { + return err + } + + _, err = io.Copy(dst, src) + if closeErr := dst.Close(); err == nil { + err = closeErr + } + return err +} + +// GroupFilesByModules attempts to map directories to the modules within each directory. +// This function assumes that the folder is structured in the following way: +// +// dir/ +// primarymod/ +// *.go files +// packages +// go.mod (optional) +// modules/ +// repoa/ +// mod1/ +// *.go files +// packages +// go.mod (optional) +// +// It scans the directory tree anchored at root and adds a Copy writer to the +// map for every file found. +// This is to enable the common case in tests where you have a full copy of the +// package in your testdata. +func GroupFilesByModules(root string) ([]Module, error) { + root = filepath.FromSlash(root) + primarymodPath := filepath.Join(root, "primarymod") + + _, err := os.Stat(primarymodPath) + if os.IsNotExist(err) { + return nil, fmt.Errorf("could not find primarymod folder within %s", root) + } + + primarymod := &Module{ + Name: root, + Files: make(map[string]interface{}), + Overlay: make(map[string][]byte), + } + mods := map[string]*Module{ + root: primarymod, + } + modules := []Module{*primarymod} + + if err := filepath.Walk(primarymodPath, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + fragment, err := filepath.Rel(primarymodPath, path) + if err != nil { + return err + } + primarymod.Files[filepath.ToSlash(fragment)] = Copy(path) + return nil + }); err != nil { + return nil, err + } + + modulesPath := filepath.Join(root, "modules") + if _, err := os.Stat(modulesPath); os.IsNotExist(err) { + return modules, nil + } + + var currentRepo, currentModule string + updateCurrentModule := func(dir string) { + if dir == currentModule { + return + } + // Handle the case where we step into a nested directory that is a module + // and then step out into the parent which is also a module. + // Example: + // - repoa + // - moda + // - go.mod + // - v2 + // - go.mod + // - what.go + // - modb + for dir != root { + if mods[dir] != nil { + currentModule = dir + return + } + dir = filepath.Dir(dir) + } + } + + if err := filepath.Walk(modulesPath, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + enclosingDir := filepath.Dir(path) + // If the path is not a directory, then we want to add the path to + // the files map of the currentModule. + if !info.IsDir() { + updateCurrentModule(enclosingDir) + fragment, err := filepath.Rel(currentModule, path) + if err != nil { + return err + } + mods[currentModule].Files[filepath.ToSlash(fragment)] = Copy(path) + return nil + } + // If the path is a directory and it's enclosing folder is equal to + // the modules folder, then the path is a new repo. + if enclosingDir == modulesPath { + currentRepo = path + return nil + } + // If the path is a directory and it's enclosing folder is not the same + // as the current repo and it is not of the form `v1`,`v2`,... + // then the path is a folder/package of the current module. + if enclosingDir != currentRepo && !versionSuffixRE.MatchString(filepath.Base(path)) { + return nil + } + // If the path is a directory and it's enclosing folder is the current repo + // then the path is a new module. + module, err := filepath.Rel(modulesPath, path) + if err != nil { + return err + } + mods[path] = &Module{ + Name: filepath.ToSlash(module), + Files: make(map[string]interface{}), + Overlay: make(map[string][]byte), + } + currentModule = path + modules = append(modules, *mods[path]) + return nil + }); err != nil { + return nil, err + } + return modules, nil +} + +// MustCopyFileTree returns a file set for a module based on a real directory tree. +// It scans the directory tree anchored at root and adds a Copy writer to the +// map for every file found. It skips copying files in nested modules. +// This is to enable the common case in tests where you have a full copy of the +// package in your testdata. +// This will panic if there is any kind of error trying to walk the file tree. +func MustCopyFileTree(root string) map[string]interface{} { + result := map[string]interface{}{} + if err := filepath.Walk(filepath.FromSlash(root), func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + // skip nested modules. + if path != root { + if fi, err := os.Stat(filepath.Join(path, "go.mod")); err == nil && !fi.IsDir() { + return filepath.SkipDir + } + } + return nil + } + fragment, err := filepath.Rel(root, path) + if err != nil { + return err + } + result[filepath.ToSlash(fragment)] = Copy(path) + return nil + }); err != nil { + log.Panic(fmt.Sprintf("MustCopyFileTree failed: %v", err)) + } + return result +} + +// Cleanup removes the temporary directory (unless the --skip-cleanup flag was set) +// It is safe to call cleanup multiple times. +func (e *Exported) Cleanup() { + if e.temp == "" { + return + } + if *skipCleanup { + log.Printf("Skipping cleanup of temp dir: %s", e.temp) + return + } + // Make everything read-write so that the Module exporter's module cache can be deleted. + filepath.Walk(e.temp, func(path string, info os.FileInfo, err error) error { + if err != nil { + return nil + } + if info.IsDir() { + os.Chmod(path, 0777) + } + return nil + }) + os.RemoveAll(e.temp) // ignore errors + e.temp = "" +} + +// Temp returns the temporary directory that was generated. +func (e *Exported) Temp() string { + return e.temp +} + +// File returns the full path for the given module and file fragment. +func (e *Exported) File(module, fragment string) string { + if m := e.written[module]; m != nil { + return m[fragment] + } + return "" +} + +// FileContents returns the contents of the specified file. +// It will use the overlay if the file is present, otherwise it will read it +// from disk. +func (e *Exported) FileContents(filename string) ([]byte, error) { + if content, found := e.Config.Overlay[filename]; found { + return content, nil + } + content, err := os.ReadFile(filename) + if err != nil { + return nil, err + } + return content, nil +} diff --git a/internal/packagestest/export_test.go b/internal/packagestest/export_test.go new file mode 100644 index 00000000000..6c074216fbe --- /dev/null +++ b/internal/packagestest/export_test.go @@ -0,0 +1,234 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packagestest_test + +import ( + "os" + "path/filepath" + "reflect" + "sort" + "testing" + + "golang.org/x/tools/internal/packagestest" +) + +var testdata = []packagestest.Module{{ + Name: "golang.org/fake1", + Files: map[string]interface{}{ + "a.go": packagestest.Symlink("testdata/a.go"), // broken symlink + "b.go": "invalid file contents", + }, + Overlay: map[string][]byte{ + "b.go": []byte("package fake1"), + "c.go": []byte("package fake1"), + }, +}, { + Name: "golang.org/fake2", + Files: map[string]interface{}{ + "other/a.go": "package fake2", + }, +}, { + Name: "golang.org/fake2/v2", + Files: map[string]interface{}{ + "other/a.go": "package fake2", + }, +}, { + Name: "golang.org/fake3@v1.0.0", + Files: map[string]interface{}{ + "other/a.go": "package fake3", + }, +}, { + Name: "golang.org/fake3@v1.1.0", + Files: map[string]interface{}{ + "other/a.go": "package fake3", + }, +}} + +type fileTest struct { + module, fragment, expect string + check func(t *testing.T, exported *packagestest.Exported, filename string) +} + +func checkFiles(t *testing.T, exported *packagestest.Exported, tests []fileTest) { + for _, test := range tests { + expect := filepath.Join(exported.Temp(), filepath.FromSlash(test.expect)) + got := exported.File(test.module, test.fragment) + if got == "" { + t.Errorf("File %v missing from the output", expect) + } else if got != expect { + t.Errorf("Got file %v, expected %v", got, expect) + } + if test.check != nil { + test.check(t, exported, got) + } + } +} + +func checkLink(expect string) func(t *testing.T, exported *packagestest.Exported, filename string) { + expect = filepath.FromSlash(expect) + return func(t *testing.T, exported *packagestest.Exported, filename string) { + if target, err := os.Readlink(filename); err != nil { + t.Errorf("Error checking link %v: %v", filename, err) + } else if target != expect { + t.Errorf("Link %v does not match, got %v expected %v", filename, target, expect) + } + } +} + +func checkContent(expect string) func(t *testing.T, exported *packagestest.Exported, filename string) { + return func(t *testing.T, exported *packagestest.Exported, filename string) { + if content, err := exported.FileContents(filename); err != nil { + t.Errorf("Error reading %v: %v", filename, err) + } else if string(content) != expect { + t.Errorf("Content of %v does not match, got %v expected %v", filename, string(content), expect) + } + } +} + +func TestGroupFilesByModules(t *testing.T) { + for _, tt := range []struct { + testdir string + want []packagestest.Module + }{ + { + testdir: "testdata/groups/one", + want: []packagestest.Module{ + { + Name: "testdata/groups/one", + Files: map[string]interface{}{ + "main.go": true, + }, + }, + { + Name: "example.com/extra", + Files: map[string]interface{}{ + "help.go": true, + }, + }, + }, + }, + { + testdir: "testdata/groups/two", + want: []packagestest.Module{ + { + Name: "testdata/groups/two", + Files: map[string]interface{}{ + "main.go": true, + "expect/yo.go": true, + "expect/yo_test.go": true, + }, + }, + { + Name: "example.com/extra", + Files: map[string]interface{}{ + "yo.go": true, + "geez/help.go": true, + }, + }, + { + Name: "example.com/extra/v2", + Files: map[string]interface{}{ + "me.go": true, + "geez/help.go": true, + }, + }, + { + Name: "example.com/tempmod", + Files: map[string]interface{}{ + "main.go": true, + }, + }, + { + Name: "example.com/what@v1.0.0", + Files: map[string]interface{}{ + "main.go": true, + }, + }, + { + Name: "example.com/what@v1.1.0", + Files: map[string]interface{}{ + "main.go": true, + }, + }, + }, + }, + } { + t.Run(tt.testdir, func(t *testing.T) { + got, err := packagestest.GroupFilesByModules(tt.testdir) + if err != nil { + t.Fatalf("could not group files %v", err) + } + if len(got) != len(tt.want) { + t.Fatalf("%s: wanted %d modules but got %d", tt.testdir, len(tt.want), len(got)) + } + for i, w := range tt.want { + g := got[i] + if filepath.FromSlash(g.Name) != filepath.FromSlash(w.Name) { + t.Fatalf("%s: wanted module[%d].Name to be %s but got %s", tt.testdir, i, filepath.FromSlash(w.Name), filepath.FromSlash(g.Name)) + } + for fh := range w.Files { + if _, ok := g.Files[fh]; !ok { + t.Fatalf("%s, module[%d]: wanted %s but could not find", tt.testdir, i, fh) + } + } + for fh := range g.Files { + if _, ok := w.Files[fh]; !ok { + t.Fatalf("%s, module[%d]: found unexpected file %s", tt.testdir, i, fh) + } + } + } + }) + } +} + +func TestMustCopyFiles(t *testing.T) { + // Create the following test directory structure in a temporary directory. + src := map[string]string{ + // copies all files under the specified directory. + "go.mod": "module example.com", + "m.go": "package m", + "a/a.go": "package a", + // contents from a nested module shouldn't be copied. + "nested/go.mod": "module example.com/nested", + "nested/m.go": "package nested", + "nested/b/b.go": "package b", + } + + tmpDir, err := os.MkdirTemp("", t.Name()) + if err != nil { + t.Fatalf("failed to create a temporary directory: %v", err) + } + defer os.RemoveAll(tmpDir) + + for fragment, contents := range src { + fullpath := filepath.Join(tmpDir, filepath.FromSlash(fragment)) + if err := os.MkdirAll(filepath.Dir(fullpath), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(fullpath, []byte(contents), 0644); err != nil { + t.Fatal(err) + } + } + + copied := packagestest.MustCopyFileTree(tmpDir) + var got []string + for fragment := range copied { + got = append(got, filepath.ToSlash(fragment)) + } + want := []string{"go.mod", "m.go", "a/a.go"} + + sort.Strings(got) + sort.Strings(want) + if !reflect.DeepEqual(got, want) { + t.Errorf("packagestest.MustCopyFileTree = %v, want %v", got, want) + } + + // packagestest.Export is happy. + exported := packagestest.Export(t, packagestest.Modules, []packagestest.Module{{ + Name: "example.com", + Files: packagestest.MustCopyFileTree(tmpDir), + }}) + defer exported.Cleanup() +} diff --git a/internal/packagestest/gopath.go b/internal/packagestest/gopath.go new file mode 100644 index 00000000000..c2e57a1545c --- /dev/null +++ b/internal/packagestest/gopath.go @@ -0,0 +1,77 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packagestest + +import ( + "path" + "path/filepath" +) + +// GOPATH is the exporter that produces GOPATH layouts. +// Each "module" is put in it's own GOPATH entry to help test complex cases. +// Given the two files +// +// golang.org/repoa#a/a.go +// golang.org/repob#b/b.go +// +// You would get the directory layout +// +// /sometemporarydirectory +// ├── repoa +// │ └── src +// │ └── golang.org +// │ └── repoa +// │ └── a +// │ └── a.go +// └── repob +// └── src +// └── golang.org +// └── repob +// └── b +// └── b.go +// +// GOPATH would be set to +// +// /sometemporarydirectory/repoa;/sometemporarydirectory/repob +// +// and the working directory would be +// +// /sometemporarydirectory/repoa/src +var GOPATH = gopath{} + +type gopath struct{} + +func (gopath) Name() string { + return "GOPATH" +} + +func (gopath) Filename(exported *Exported, module, fragment string) string { + return filepath.Join(gopathDir(exported, module), "src", module, fragment) +} + +func (gopath) Finalize(exported *Exported) error { + exported.Config.Env = append(exported.Config.Env, "GO111MODULE=off") + gopath := "" + for module := range exported.written { + if gopath != "" { + gopath += string(filepath.ListSeparator) + } + dir := gopathDir(exported, module) + gopath += dir + if module == exported.primary { + exported.Config.Dir = filepath.Join(dir, "src") + } + } + exported.Config.Env = append(exported.Config.Env, "GOPATH="+gopath) + return nil +} + +func gopathDir(exported *Exported, module string) string { + dir := path.Base(module) + if versionSuffixRE.MatchString(dir) { + dir = path.Base(path.Dir(module)) + "_" + dir + } + return filepath.Join(exported.temp, dir) +} diff --git a/internal/packagestest/gopath_test.go b/internal/packagestest/gopath_test.go new file mode 100644 index 00000000000..fa9f7e545eb --- /dev/null +++ b/internal/packagestest/gopath_test.go @@ -0,0 +1,28 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packagestest_test + +import ( + "path/filepath" + "testing" + + "golang.org/x/tools/internal/packagestest" +) + +func TestGOPATHExport(t *testing.T) { + exported := packagestest.Export(t, packagestest.GOPATH, testdata) + defer exported.Cleanup() + // Check that the cfg contains all the right bits + var expectDir = filepath.Join(exported.Temp(), "fake1", "src") + if exported.Config.Dir != expectDir { + t.Errorf("Got working directory %v expected %v", exported.Config.Dir, expectDir) + } + checkFiles(t, exported, []fileTest{ + {"golang.org/fake1", "a.go", "fake1/src/golang.org/fake1/a.go", checkLink("testdata/a.go")}, + {"golang.org/fake1", "b.go", "fake1/src/golang.org/fake1/b.go", checkContent("package fake1")}, + {"golang.org/fake2", "other/a.go", "fake2/src/golang.org/fake2/other/a.go", checkContent("package fake2")}, + {"golang.org/fake2/v2", "other/a.go", "fake2_v2/src/golang.org/fake2/v2/other/a.go", checkContent("package fake2")}, + }) +} diff --git a/internal/packagestest/modules.go b/internal/packagestest/modules.go new file mode 100644 index 00000000000..0c8d3d8fec9 --- /dev/null +++ b/internal/packagestest/modules.go @@ -0,0 +1,223 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packagestest + +import ( + "bytes" + "context" + "fmt" + "os" + "path" + "path/filepath" + "regexp" + "strings" + + "golang.org/x/tools/internal/gocommand" + "golang.org/x/tools/internal/proxydir" +) + +// Modules is the exporter that produces module layouts. +// Each "repository" is put in its own module, and the module file generated +// will have replace directives for all other modules. +// Given the two files +// +// golang.org/repoa#a/a.go +// golang.org/repob#b/b.go +// +// You would get the directory layout +// +// /sometemporarydirectory +// ├── repoa +// │ ├── a +// │ │ └── a.go +// │ └── go.mod +// └── repob +// ├── b +// │ └── b.go +// └── go.mod +// +// and the working directory would be +// +// /sometemporarydirectory/repoa +var Modules = modules{} + +type modules struct{} + +type moduleAtVersion struct { + module string + version string +} + +func (modules) Name() string { + return "Modules" +} + +func (modules) Filename(exported *Exported, module, fragment string) string { + if module == exported.primary { + return filepath.Join(primaryDir(exported), fragment) + } + return filepath.Join(moduleDir(exported, module), fragment) +} + +func (modules) Finalize(exported *Exported) error { + // Write out the primary module. This module can use symlinks and + // other weird stuff, and will be the working dir for the go command. + // It depends on all the other modules. + primaryDir := primaryDir(exported) + if err := os.MkdirAll(primaryDir, 0755); err != nil { + return err + } + exported.Config.Dir = primaryDir + if exported.written[exported.primary] == nil { + exported.written[exported.primary] = make(map[string]string) + } + + // Create a map of modulepath -> {module, version} for modulepaths + // that are of the form `repoa/mod1@v1.1.0`. + versions := make(map[string]moduleAtVersion) + for module := range exported.written { + if splt := strings.Split(module, "@"); len(splt) > 1 { + versions[module] = moduleAtVersion{ + module: splt[0], + version: splt[1], + } + } + } + + // If the primary module already has a go.mod, write the contents to a temp + // go.mod for now and then we will reset it when we are getting all the markers. + if gomod := exported.written[exported.primary]["go.mod"]; gomod != "" { + contents, err := os.ReadFile(gomod) + if err != nil { + return err + } + if err := os.WriteFile(gomod+".temp", contents, 0644); err != nil { + return err + } + } + + exported.written[exported.primary]["go.mod"] = filepath.Join(primaryDir, "go.mod") + var primaryGomod bytes.Buffer + fmt.Fprintf(&primaryGomod, "module %s\nrequire (\n", exported.primary) + for other := range exported.written { + if other == exported.primary { + continue + } + version := moduleVersion(other) + // If other is of the form `repo1/mod1@v1.1.0`, + // then we need to extract the module and the version. + if v, ok := versions[other]; ok { + other = v.module + version = v.version + } + fmt.Fprintf(&primaryGomod, "\t%v %v\n", other, version) + } + fmt.Fprintf(&primaryGomod, ")\n") + if err := os.WriteFile(filepath.Join(primaryDir, "go.mod"), primaryGomod.Bytes(), 0644); err != nil { + return err + } + + // Create the mod cache so we can rename it later, even if we don't need it. + if err := os.MkdirAll(modCache(exported), 0755); err != nil { + return err + } + + // Write out the go.mod files for the other modules. + for module, files := range exported.written { + if module == exported.primary { + continue + } + dir := moduleDir(exported, module) + modfile := filepath.Join(dir, "go.mod") + // If other is of the form `repo1/mod1@v1.1.0`, + // then we need to extract the module name without the version. + if v, ok := versions[module]; ok { + module = v.module + } + if err := os.WriteFile(modfile, []byte("module "+module+"\n"), 0644); err != nil { + return err + } + files["go.mod"] = modfile + } + + // Zip up all the secondary modules into the proxy dir. + modProxyDir := filepath.Join(exported.temp, "modproxy") + for module, files := range exported.written { + if module == exported.primary { + continue + } + version := moduleVersion(module) + // If other is of the form `repo1/mod1@v1.1.0`, + // then we need to extract the module and the version. + if v, ok := versions[module]; ok { + module = v.module + version = v.version + } + if err := writeModuleFiles(modProxyDir, module, version, files); err != nil { + return fmt.Errorf("creating module proxy dir for %v: %v", module, err) + } + } + + // Discard the original mod cache dir, which contained the files written + // for us by Export. + if err := os.Rename(modCache(exported), modCache(exported)+".orig"); err != nil { + return err + } + exported.Config.Env = append(exported.Config.Env, + "GO111MODULE=on", + "GOPATH="+filepath.Join(exported.temp, "modcache"), + "GOMODCACHE=", + "GOPROXY="+proxydir.ToURL(modProxyDir), + "GOSUMDB=off", + ) + + // Run go mod download to recreate the mod cache dir with all the extra + // stuff in cache. All the files created by Export should be recreated. + inv := gocommand.Invocation{ + Verb: "mod", + Args: []string{"download", "all"}, + Env: exported.Config.Env, + BuildFlags: exported.Config.BuildFlags, + WorkingDir: exported.Config.Dir, + } + _, err := new(gocommand.Runner).Run(context.Background(), inv) + return err +} + +func writeModuleFiles(rootDir, module, ver string, filePaths map[string]string) error { + fileData := make(map[string][]byte) + for name, path := range filePaths { + contents, err := os.ReadFile(path) + if err != nil { + return err + } + fileData[name] = contents + } + return proxydir.WriteModuleVersion(rootDir, module, ver, fileData) +} + +func modCache(exported *Exported) string { + return filepath.Join(exported.temp, "modcache/pkg/mod") +} + +func primaryDir(exported *Exported) string { + return filepath.Join(exported.temp, path.Base(exported.primary)) +} + +func moduleDir(exported *Exported, module string) string { + if strings.Contains(module, "@") { + return filepath.Join(modCache(exported), module) + } + return filepath.Join(modCache(exported), path.Dir(module), path.Base(module)+"@"+moduleVersion(module)) +} + +var versionSuffixRE = regexp.MustCompile(`v\d+`) + +func moduleVersion(module string) string { + if versionSuffixRE.MatchString(path.Base(module)) { + return path.Base(module) + ".0.0" + } + return "v1.0.0" +} diff --git a/internal/packagestest/modules_test.go b/internal/packagestest/modules_test.go new file mode 100644 index 00000000000..a1beeed7ac3 --- /dev/null +++ b/internal/packagestest/modules_test.go @@ -0,0 +1,32 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packagestest_test + +import ( + "path/filepath" + "testing" + + "golang.org/x/tools/internal/packagestest" +) + +func TestModulesExport(t *testing.T) { + exported := packagestest.Export(t, packagestest.Modules, testdata) + defer exported.Cleanup() + // Check that the cfg contains all the right bits + var expectDir = filepath.Join(exported.Temp(), "fake1") + if exported.Config.Dir != expectDir { + t.Errorf("Got working directory %v expected %v", exported.Config.Dir, expectDir) + } + checkFiles(t, exported, []fileTest{ + {"golang.org/fake1", "go.mod", "fake1/go.mod", nil}, + {"golang.org/fake1", "a.go", "fake1/a.go", checkLink("testdata/a.go")}, + {"golang.org/fake1", "b.go", "fake1/b.go", checkContent("package fake1")}, + {"golang.org/fake2", "go.mod", "modcache/pkg/mod/golang.org/fake2@v1.0.0/go.mod", nil}, + {"golang.org/fake2", "other/a.go", "modcache/pkg/mod/golang.org/fake2@v1.0.0/other/a.go", checkContent("package fake2")}, + {"golang.org/fake2/v2", "other/a.go", "modcache/pkg/mod/golang.org/fake2/v2@v2.0.0/other/a.go", checkContent("package fake2")}, + {"golang.org/fake3@v1.1.0", "other/a.go", "modcache/pkg/mod/golang.org/fake3@v1.1.0/other/a.go", checkContent("package fake3")}, + {"golang.org/fake3@v1.0.0", "other/a.go", "modcache/pkg/mod/golang.org/fake3@v1.0.0/other/a.go", nil}, + }) +} diff --git a/internal/packagestest/testdata/groups/one/modules/example.com/extra/help.go b/internal/packagestest/testdata/groups/one/modules/example.com/extra/help.go new file mode 100644 index 00000000000..ee032937550 --- /dev/null +++ b/internal/packagestest/testdata/groups/one/modules/example.com/extra/help.go @@ -0,0 +1 @@ +package extra \ No newline at end of file diff --git a/internal/packagestest/testdata/groups/one/primarymod/main.go b/internal/packagestest/testdata/groups/one/primarymod/main.go new file mode 100644 index 00000000000..54fe6e8b326 --- /dev/null +++ b/internal/packagestest/testdata/groups/one/primarymod/main.go @@ -0,0 +1 @@ +package one \ No newline at end of file diff --git a/internal/packagestest/testdata/groups/two/modules/example.com/extra/geez/help.go b/internal/packagestest/testdata/groups/two/modules/example.com/extra/geez/help.go new file mode 100644 index 00000000000..930ffdc81fe --- /dev/null +++ b/internal/packagestest/testdata/groups/two/modules/example.com/extra/geez/help.go @@ -0,0 +1 @@ +package example.com/extra/geez \ No newline at end of file diff --git a/internal/packagestest/testdata/groups/two/modules/example.com/extra/v2/geez/help.go b/internal/packagestest/testdata/groups/two/modules/example.com/extra/v2/geez/help.go new file mode 100644 index 00000000000..930ffdc81fe --- /dev/null +++ b/internal/packagestest/testdata/groups/two/modules/example.com/extra/v2/geez/help.go @@ -0,0 +1 @@ +package example.com/extra/geez \ No newline at end of file diff --git a/internal/packagestest/testdata/groups/two/modules/example.com/extra/v2/me.go b/internal/packagestest/testdata/groups/two/modules/example.com/extra/v2/me.go new file mode 100644 index 00000000000..6a8c7d31f24 --- /dev/null +++ b/internal/packagestest/testdata/groups/two/modules/example.com/extra/v2/me.go @@ -0,0 +1 @@ +package example.com/extra \ No newline at end of file diff --git a/internal/packagestest/testdata/groups/two/modules/example.com/extra/yo.go b/internal/packagestest/testdata/groups/two/modules/example.com/extra/yo.go new file mode 100644 index 00000000000..6a8c7d31f24 --- /dev/null +++ b/internal/packagestest/testdata/groups/two/modules/example.com/extra/yo.go @@ -0,0 +1 @@ +package example.com/extra \ No newline at end of file diff --git a/internal/packagestest/testdata/groups/two/modules/example.com/tempmod/main.go b/internal/packagestest/testdata/groups/two/modules/example.com/tempmod/main.go new file mode 100644 index 00000000000..85dbfa7cf31 --- /dev/null +++ b/internal/packagestest/testdata/groups/two/modules/example.com/tempmod/main.go @@ -0,0 +1 @@ +package example.com/tempmod \ No newline at end of file diff --git a/internal/packagestest/testdata/groups/two/modules/example.com/what@v1.0.0/main.go b/internal/packagestest/testdata/groups/two/modules/example.com/what@v1.0.0/main.go new file mode 100644 index 00000000000..4723ee64bb1 --- /dev/null +++ b/internal/packagestest/testdata/groups/two/modules/example.com/what@v1.0.0/main.go @@ -0,0 +1 @@ +package example.com/what \ No newline at end of file diff --git a/internal/packagestest/testdata/groups/two/modules/example.com/what@v1.1.0/main.go b/internal/packagestest/testdata/groups/two/modules/example.com/what@v1.1.0/main.go new file mode 100644 index 00000000000..4723ee64bb1 --- /dev/null +++ b/internal/packagestest/testdata/groups/two/modules/example.com/what@v1.1.0/main.go @@ -0,0 +1 @@ +package example.com/what \ No newline at end of file diff --git a/internal/packagestest/testdata/groups/two/primarymod/expect/yo.go b/internal/packagestest/testdata/groups/two/primarymod/expect/yo.go new file mode 100644 index 00000000000..bce2d30e094 --- /dev/null +++ b/internal/packagestest/testdata/groups/two/primarymod/expect/yo.go @@ -0,0 +1,3 @@ +package expect + +var X int //@check("X", "X") diff --git a/internal/packagestest/testdata/groups/two/primarymod/expect/yo_test.go b/internal/packagestest/testdata/groups/two/primarymod/expect/yo_test.go new file mode 100644 index 00000000000..a8b06126582 --- /dev/null +++ b/internal/packagestest/testdata/groups/two/primarymod/expect/yo_test.go @@ -0,0 +1,10 @@ +package expect_test + +import ( + "testdata/groups/two/expect" + "testing" +) + +func TestX(t *testing.T) { + _ = expect.X //@check("X", "X") +} diff --git a/internal/packagestest/testdata/groups/two/primarymod/main.go b/internal/packagestest/testdata/groups/two/primarymod/main.go new file mode 100644 index 00000000000..0b263348651 --- /dev/null +++ b/internal/packagestest/testdata/groups/two/primarymod/main.go @@ -0,0 +1 @@ +package two \ No newline at end of file diff --git a/internal/packagestest/testdata/test.go b/internal/packagestest/testdata/test.go new file mode 100644 index 00000000000..13fc12b9fae --- /dev/null +++ b/internal/packagestest/testdata/test.go @@ -0,0 +1,24 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fake1 + +// This is a test file for the behaviors in Exported.Expect. + +type AThing string //@AThing,mark(StringThing, "AThing"),mark(REThing,re`.T.*g`) + +type Match string //@check("Match",re`[[:upper:]]`) + +//@check(AThing, StringThing) +//@check(AThing, REThing) + +//@boolArg(true, false) +//@intArg(42) +//@stringArg(PlainString, "PlainString") +//@stringArg(IdentAsString,IdentAsString) +//@directNote() +//@range(AThing) + +// The following test should remain at the bottom of the file +//@checkEOF(EOF) diff --git a/internal/packagestest/testdata/test_test.go b/internal/packagestest/testdata/test_test.go new file mode 100644 index 00000000000..18b20805f95 --- /dev/null +++ b/internal/packagestest/testdata/test_test.go @@ -0,0 +1,3 @@ +package fake1 + +type ATestType string //@check("ATestType","ATestType") diff --git a/internal/packagestest/testdata/x_test.go b/internal/packagestest/testdata/x_test.go new file mode 100644 index 00000000000..c8c4fa25343 --- /dev/null +++ b/internal/packagestest/testdata/x_test.go @@ -0,0 +1,3 @@ +package fake1_test + +type AnXTestType string //@check("AnXTestType","AnXTestType") diff --git a/refactor/importgraph/graph_test.go b/refactor/importgraph/graph_test.go index eeb7b054980..f3378a41e86 100644 --- a/refactor/importgraph/graph_test.go +++ b/refactor/importgraph/graph_test.go @@ -17,7 +17,7 @@ import ( "strings" "testing" - "golang.org/x/tools/go/packages/packagestest" + "golang.org/x/tools/internal/packagestest" "golang.org/x/tools/refactor/importgraph" _ "crypto/hmac" // just for test, below