cmd/compile: add indexed export format

This CL introduces a new indexed data format for package export
data. This improves on the previous (sequential) binary format by
allowing the compiler to selectively (and lazily) load only the data
that's actually needed for compilation.

In large Go projects, the package export data can become very large
due to transitive type declaration dependencies and inline
function/method bodies. By lazily loading these declarations and
bodies as needed, we avoid wasting time and memory processing
unnecessary and/or redundant data.

In the benchmarks below, "old" is -iexport=false and "new" is
-iexport=true. The suffixes indicate the compiler concurrency (-c) and
inlining (-l) settings used for the build (using -gcflags=all=-foo).
Benchmarks were run on an HP Z620.

Juju is "go build -a github.com/juju/juju/cmd/...":

name          old real-time/op  new real-time/op  delta
Juju/c=1/l=0        44.0s ± 1%        38.7s ± 9%  -11.97%  (p=0.001 n=7+7)
Juju/c=1/l=4        53.7s ± 3%        45.3s ± 4%  -15.53%  (p=0.001 n=7+7)
Juju/c=4/l=0        39.7s ± 8%        32.0s ± 4%  -19.38%  (p=0.001 n=7+7)
Juju/c=4/l=4        46.3s ± 4%        38.0s ± 4%  -18.06%  (p=0.001 n=7+7)

name          old user-time/op  new user-time/op  delta
Juju/c=1/l=0         371s ± 1%         300s ± 0%  -19.07%  (p=0.001 n=7+6)
Juju/c=1/l=4         482s ± 0%         374s ± 1%  -22.37%  (p=0.001 n=7+7)
Juju/c=4/l=0         410s ± 1%         340s ± 1%  -17.19%  (p=0.001 n=7+7)
Juju/c=4/l=4         532s ± 1%         424s ± 1%  -20.26%  (p=0.001 n=7+7)

name          old sys-time/op   new sys-time/op   delta
Juju/c=1/l=0        33.4s ± 1%        28.4s ± 2%  -15.02%  (p=0.001 n=7+7)
Juju/c=1/l=4        40.7s ± 2%        32.8s ± 3%  -19.51%  (p=0.001 n=7+7)
Juju/c=4/l=0        39.8s ± 2%        34.4s ± 2%  -13.74%  (p=0.001 n=7+7)
Juju/c=4/l=4        48.4s ± 2%        40.4s ± 2%  -16.50%  (p=0.001 n=7+7)

Kubelet is "go build -a k8s.io/kubernetes/cmd/kubelet":

name             old real-time/op  new real-time/op  delta
Kubelet/c=1/l=0        42.0s ± 1%        34.8s ± 1%  -17.27%  (p=0.008 n=5+5)
Kubelet/c=1/l=4        55.4s ± 3%        45.4s ± 3%  -18.06%  (p=0.002 n=6+6)
Kubelet/c=4/l=0        37.4s ± 3%        29.9s ± 1%  -20.25%  (p=0.004 n=6+5)
Kubelet/c=4/l=4        48.1s ± 2%        39.0s ± 5%  -18.93%  (p=0.002 n=6+6)

name             old user-time/op  new user-time/op  delta
Kubelet/c=1/l=0         291s ± 1%         233s ± 1%  -19.96%  (p=0.002 n=6+6)
Kubelet/c=1/l=4         385s ± 1%         298s ± 1%  -22.51%  (p=0.002 n=6+6)
Kubelet/c=4/l=0         325s ± 0%         268s ± 1%  -17.48%  (p=0.004 n=5+6)
Kubelet/c=4/l=4         429s ± 1%         343s ± 1%  -20.08%  (p=0.002 n=6+6)

name             old sys-time/op   new sys-time/op   delta
Kubelet/c=1/l=0        25.1s ± 2%        20.9s ± 4%  -16.69%  (p=0.002 n=6+6)
Kubelet/c=1/l=4        31.2s ± 3%        24.4s ± 0%  -21.67%  (p=0.010 n=6+4)
Kubelet/c=4/l=0        30.2s ± 2%        25.6s ± 1%  -15.34%  (p=0.002 n=6+6)
Kubelet/c=4/l=4        37.3s ± 1%        30.9s ± 2%  -17.11%  (p=0.002 n=6+6)

Change-Id: Ie43eb3bbe1392cbb61c86792a17a57b33b9561f0
Reviewed-on: https://go-review.googlesource.com/106796
Run-TryBot: Matthew Dempsky <mdempsky@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Robert Griesemer <gri@golang.org>

src/cmd/compile/fmt_test.go

@@ -598,6 +598,7 @@ var knownFormats = map[string]string{
 	"*cmd/internal/obj.LSym %v":                       "",
 	"*math/big.Int %#x":                               "",
 	"*math/big.Int %s":                                "",
+	"*math/big.Int %v":                                "",
 	"[16]byte %x":                                     "",
 	"[]*cmd/compile/internal/gc.Node %v":              "",
 	"[]*cmd/compile/internal/ssa.Block %v":            "",
@@ -612,6 +613,7 @@ var knownFormats = map[string]string{
 	"bool %v":                                         "",
 	"byte %08b":                                       "",
 	"byte %c":                                         "",
+	"byte %v":                                         "",
 	"cmd/compile/internal/arm.shift %d":               "",
 	"cmd/compile/internal/gc.Class %d":                "",
 	"cmd/compile/internal/gc.Class %s":                "",
@@ -631,6 +633,7 @@ var knownFormats = map[string]string{
 	"cmd/compile/internal/gc.Val %v":                  "",
 	"cmd/compile/internal/gc.fmtMode %d":              "",
 	"cmd/compile/internal/gc.initKind %d":             "",
+	"cmd/compile/internal/gc.itag %v":                 "",
 	"cmd/compile/internal/ssa.BranchPrediction %d":    "",
 	"cmd/compile/internal/ssa.Edge %v":                "",
 	"cmd/compile/internal/ssa.GCNode %v":              "",

src/cmd/compile/internal/gc/closure.go

@@ -442,6 +442,8 @@ func makepartialcall(fn *Node, t0 *types.Type, meth *types.Sym) *Node {
 	xfunc.Func.SetDupok(true)
 	xfunc.Func.SetNeedctxt(true)
 
+	tfn.Type.SetPkg(t0.Pkg())
+
 	// Declare and initialize variable holding receiver.
 
 	cv := nod(OCLOSUREVAR, nil, nil)

src/cmd/compile/internal/gc/export.go

@@ -5,15 +5,15 @@
 package gc
 
 import (
-	"bufio"
-	"bytes"
 	"cmd/compile/internal/types"
 	"cmd/internal/bio"
 	"cmd/internal/src"
 	"fmt"
 )
 
 var (
+	flagiexport bool // if set, use indexed export data format
+
 	Debug_export int // if set, print debugging information about export data
 )
 
@@ -72,32 +72,15 @@ func (x methodbyname) Swap(i, j int)      { x[i], x[j] = x[j], x[i] }
 func (x methodbyname) Less(i, j int) bool { return x[i].Sym.Name < x[j].Sym.Name }
 
 func dumpexport(bout *bio.Writer) {
-	size := 0 // size of export section without enclosing markers
 	// The linker also looks for the $$ marker - use char after $$ to distinguish format.
 	exportf(bout, "\n$$B\n") // indicate binary export format
-	if debugFormat {
-		// save a copy of the export data
-		var copy bytes.Buffer
-		bcopy := bufio.NewWriter(&copy)
-		size = export(bcopy, Debug_export != 0)
-		bcopy.Flush() // flushing to bytes.Buffer cannot fail
-		if n, err := bout.Write(copy.Bytes()); n != size || err != nil {
-			Fatalf("error writing export data: got %d bytes, want %d bytes, err = %v", n, size, err)
-		}
-		// export data must contain no '$' so that we can find the end by searching for "$$"
-		// TODO(gri) is this still needed?
-		if bytes.IndexByte(copy.Bytes(), '$') >= 0 {
-			Fatalf("export data contains $")
-		}
-
-		// verify that we can read the copied export data back in
-		// (use empty package map to avoid collisions)
-		types.CleanroomDo(func() {
-			Import(types.NewPkg("", ""), bufio.NewReader(&copy)) // must not die
-		})
+	off := bout.Offset()
+	if flagiexport {
+		iexport(bout.Writer)
 	} else {
-		size = export(bout.Writer, Debug_export != 0)
+		export(bout.Writer, Debug_export != 0)
 	}
+	size := bout.Offset() - off
 	exportf(bout, "\n$$\n")
 
 	if Debug_export != 0 {
@@ -108,6 +91,14 @@ func dumpexport(bout *bio.Writer) {
 func importsym(ipkg *types.Pkg, pos src.XPos, s *types.Sym, op Op) *Node {
 	n := asNode(s.Def)
 	if n == nil {
+		// iimport should have created a stub ONONAME
+		// declaration for all imported symbols. The exception
+		// is declarations for Runtimepkg, which are populated
+		// by loadsys instead.
+		if flagiexport && s.Pkg != Runtimepkg {
+			Fatalf("missing ONONAME for %v\n", s)
+		}
+
 		n = dclname(s)
 		s.Def = asTypesNode(n)
 		s.Importdef = ipkg