Skip to content

Commit

Permalink
internal/gcimporter: rewrite TestIExportData_stdlib using go/packages
Browse files Browse the repository at this point in the history 
This breaks a dependency on the deprecated go/loader,
and removes a bunch of legacy cruft.

Updates golang/go#69556

Change-Id: I13e1b249db761bd0604881233116ca2a6c9c9904
Reviewed-on: https://go-review.googlesource.com/c/tools/+/614676
Auto-Submit: Alan Donovan <adonovan@google.com>
Reviewed-by: Tim King <taking@google.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
  • Loading branch information
@adonovan @gopherbot
adonovan authored and gopherbot committed Sep 22, 2024
1 parent af67b9d commit d2c1642
Show file tree
Hide file tree
Showing 2 changed files with 67 additions and 109 deletions.
4 changes: 0 additions & 4 deletions internal/gcimporter/iexport_go118_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,10 +106,6 @@ func TestImportTypeparamTests(t *testing.T) {
t.Fatal(err)
}

if isUnifiedBuilder() {
t.Skip("unified export data format is currently unsupported")
}

for _, entry := range list {
if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".go") {
// For now, only consider standalone go files.
Expand Down
172 changes: 67 additions & 105 deletions internal/gcimporter/iexport_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,57 +7,28 @@
package gcimporter_test

import (
"bufio"
"bytes"
"fmt"
"go/ast"
"go/build"
"go/constant"
"go/importer"
"go/parser"
"go/token"
"go/types"
"io"
"math/big"
"os"
"path/filepath"
"reflect"
"runtime"
"sort"
"strings"
"testing"

"golang.org/x/tools/go/ast/inspector"
"golang.org/x/tools/go/buildutil"
"golang.org/x/tools/go/gcexportdata"
"golang.org/x/tools/go/loader"
"golang.org/x/tools/go/packages"
"golang.org/x/tools/internal/aliases"
"golang.org/x/tools/internal/gcimporter"
"golang.org/x/tools/internal/testenv"
"golang.org/x/tools/internal/typeparams/genericfeatures"
)

func readExportFile(filename string) ([]byte, error) {
f, err := os.Open(filename)
if err != nil {
return nil, err
}
defer f.Close()

buf := bufio.NewReader(f)
if _, _, err := gcimporter.FindExportData(buf); err != nil {
return nil, err
}

if ch, err := buf.ReadByte(); err != nil {
return nil, err
} else if ch != 'i' {
return nil, fmt.Errorf("unexpected byte: %v", ch)
}

return io.ReadAll(buf)
}

func iexport(fset *token.FileSet, version int, pkg *types.Package) ([]byte, error) {
var buf bytes.Buffer
const bundle, shallow = false, false
Expand All @@ -67,105 +38,95 @@ func iexport(fset *token.FileSet, version int, pkg *types.Package) ([]byte, erro
return buf.Bytes(), nil
}

// isUnifiedBuilder reports whether we are executing on a go builder that uses
// unified export data.
func isUnifiedBuilder() bool {
return os.Getenv("GO_BUILDER_NAME") == "linux-amd64-unified"
}

const minStdlibPackages = 248

func TestIExportData_stdlib(t *testing.T) {
if runtime.Compiler == "gccgo" {
t.Skip("gccgo standard library is inaccessible")
}
testenv.NeedsGoBuild(t)
testenv.NeedsGoPackages(t)
if isRace {
t.Skipf("stdlib tests take too long in race mode and flake on builders")
}
if testing.Short() {
t.Skip("skipping RAM hungry test in -short mode")
}

// Load, parse and type-check the program.
ctxt := build.Default // copy
ctxt.GOPATH = "" // disable GOPATH
conf := loader.Config{
Build: &ctxt,
AllowErrors: true,
TypeChecker: types.Config{
Sizes: types.SizesFor(ctxt.Compiler, ctxt.GOARCH),
Error: func(err error) { t.Log(err) },
},
}
for _, path := range buildutil.AllPackages(conf.Build) {
conf.Import(path)
var errorsDir string // GOROOT/src/errors directory
{
cfg := packages.Config{
Mode: packages.NeedName | packages.NeedFiles,
}
pkgs, err := packages.Load(&cfg, "errors")
if err != nil {
t.Fatal(err)
}
errorsDir = filepath.Dir(pkgs[0].GoFiles[0])
}

// Create a package containing type and value errors to ensure
// they are properly encoded/decoded.
f, err := conf.ParseFile("haserrors/haserrors.go", `package haserrors
// Load types from syntax for all std packages.
//
// Append a file to package errors containing type and
// value errors to ensure they are properly encoded/decoded.
const bad = `package errors
const UnknownValue = "" + 0
type UnknownType undefined
`)
`
cfg := packages.Config{
Mode: packages.LoadAllSyntax | packages.NeedDeps,
Overlay: map[string][]byte{filepath.Join(errorsDir, "bad.go"): []byte(bad)},
}
pkgs, err := packages.Load(&cfg, "std") // ~800ms
if err != nil {
t.Fatal(err)
}
conf.CreateFromFiles("haserrors", f)
fset := pkgs[0].Fset

prog, err := conf.Load()
if err != nil {
t.Fatalf("Load failed: %v", err)
}
version := gcimporter.IExportVersion

var sorted []*types.Package
isUnified := isUnifiedBuilder()
for pkg, info := range prog.AllPackages {
// Temporarily skip packages that use generics on the unified builder, to
// fix TryBots.
//
// TODO(#48595): fix this test with GOEXPERIMENT=unified.
inspect := inspector.New(info.Files)
features := genericfeatures.ForPackage(inspect, &info.Info)
if isUnified && features != 0 {
t.Logf("skipping package %q which uses generics", pkg.Path())
continue
// Export and reimport each package, and check that they match.
var allPkgs []*types.Package
var errorsPkg *types.Package // reimported errors package
packages.Visit(pkgs, nil, func(ppkg *packages.Package) { // ~300ms
pkg := ppkg.Types
path := pkg.Path()
if path == "unsafe" ||
strings.HasPrefix(path, "cmd/") ||
strings.HasPrefix(path, "vendor/") {
return
}
if info.Files != nil { // non-empty directory
sorted = append(sorted, pkg)
allPkgs = append(allPkgs, pkg)

// Export and reimport the package, and compare.
exportdata, err := iexport(fset, version, pkg)
if err != nil {
t.Error(err)
return
}
pkg2 := testPkgData(t, fset, version, pkg, exportdata)
if path == "errors" {
errorsPkg = pkg2
}
}
sort.Slice(sorted, func(i, j int) bool {
return sorted[i].Path() < sorted[j].Path()
})

version := gcimporter.IExportVersion
numPkgs := len(sorted)
if want := minStdlibPackages; numPkgs < want {
t.Errorf("Loaded only %d packages, want at least %d", numPkgs, want)
// Assert that we saw a plausible sized library.
const minStdlibPackages = 284
if n := len(allPkgs); n < minStdlibPackages {
t.Errorf("Loaded only %d packages, want at least %d", n, minStdlibPackages)
}

// TODO(adonovan): opt: parallelize this slow loop.
for _, pkg := range sorted {
if exportdata, err := iexport(conf.Fset, version, pkg); err != nil {
t.Error(err)
} else {
testPkgData(t, conf.Fset, version, pkg, exportdata)
// Check that reimported errors package has bad decls.
if errorsPkg == nil {
t.Fatalf("'errors' package not found")
}
for _, name := range []string{"UnknownType", "UnknownValue"} {
obj := errorsPkg.Scope().Lookup(name)
if obj == nil {
t.Errorf("errors.%s not found", name)
}

if pkg.Name() == "main" || pkg.Name() == "haserrors" {
// skip; no export data
} else if bp, err := ctxt.Import(pkg.Path(), "", build.FindOnly); err != nil {
t.Log("warning:", err)
} else if exportdata, err := readExportFile(bp.PkgObj); err != nil {
t.Log("warning:", err)
} else {
testPkgData(t, conf.Fset, version, pkg, exportdata)
if typ := obj.Type().Underlying(); typ.String() != "invalid type" {
t.Errorf("errors.%s has underlying type %s, want invalid type", name, typ)
}
}

// (Sole) test of bundle functionality (250ms).
var bundle bytes.Buffer
if err := gcimporter.IExportBundle(&bundle, conf.Fset, sorted); err != nil {
if err := gcimporter.IExportBundle(&bundle, fset, allPkgs); err != nil {
t.Fatal(err)
}
fset2 := token.NewFileSet()
Expand All @@ -174,13 +135,13 @@ type UnknownType undefined
if err != nil {
t.Fatal(err)
}

for i, pkg := range sorted {
testPkg(t, conf.Fset, version, pkg, fset2, pkgs2[i])
for i, pkg := range allPkgs {
testPkg(t, fset, version, pkg, fset2, pkgs2[i])
}
}

func testPkgData(t *testing.T, fset *token.FileSet, version int, pkg *types.Package, exportdata []byte) {
// testPkgData imports a package from export data and compares it with pkg.
func testPkgData(t *testing.T, fset *token.FileSet, version int, pkg *types.Package, exportdata []byte) *types.Package {
imports := make(map[string]*types.Package)
fset2 := token.NewFileSet()
_, pkg2, err := gcimporter.IImportData(fset2, imports, exportdata, pkg.Path())
Expand All @@ -189,6 +150,7 @@ func testPkgData(t *testing.T, fset *token.FileSet, version int, pkg *types.Pack
}

testPkg(t, fset, version, pkg, fset2, pkg2)
return pkg2
}

func testPkg(t *testing.T, fset *token.FileSet, version int, pkg *types.Package, fset2 *token.FileSet, pkg2 *types.Package) {
Expand Down

0 comments on commit d2c1642

Please sign in to comment.