Skip to content

Commit

Permalink
builder: use build ID as cache key
Browse files Browse the repository at this point in the history
Instead of storing an increasing version number in relevant packages
(compiler.Version, interp.Version, cgo.Version, ...), read the build ID
from the currently running executable. This has several benefits:

  * All changes relevant to the compiled packages are caught.
  * No need to bump the version for each change to these packages.
    This avoids merge conflicts.
  * During development, `go install` is enough. No need to run
    `tinygo clean` all the time.

Of course, the drawback is that it might be updated a bit more often
than necessary but I think the overall benefit is big.

Regular release users shouldn't see any difference. Because the tinygo
binary stays the same, the cache works well.
  • Loading branch information
aykevl authored and niaow committed Dec 28, 2021
1 parent 763a86c commit 3e109fc
Show file tree
Hide file tree
Showing 5 changed files with 103 additions and 24 deletions.
18 changes: 11 additions & 7 deletions builder/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ import (
"strings"

"github.com/gofrs/flock"
"github.com/tinygo-org/tinygo/cgo"
"github.com/tinygo-org/tinygo/compileopts"
"github.com/tinygo-org/tinygo/compiler"
"github.com/tinygo-org/tinygo/goenv"
Expand Down Expand Up @@ -64,9 +63,8 @@ type BuildResult struct {
// implementation of an imported package changes.
type packageAction struct {
ImportPath string
CGoVersion int // cgo.Version
CompilerVersion int // compiler.Version
InterpVersion int // interp.Version
CompilerBuildID string
TinyGoVersion string
LLVMVersion string
Config *compiler.Config
CFlags []string
Expand All @@ -84,6 +82,13 @@ type packageAction struct {
// The error value may be of type *MultiError. Callers will likely want to check
// for this case and print such errors individually.
func Build(pkgName, outpath string, config *compileopts.Config, action func(BuildResult) error) error {
// Read the build ID of the tinygo binary.
// Used as a cache key for package builds.
compilerBuildID, err := ReadBuildID()
if err != nil {
return err
}

// Create a temporary directory for intermediary files.
dir, err := ioutil.TempDir("", "tinygo")
if err != nil {
Expand Down Expand Up @@ -192,9 +197,8 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil
// the parameters for the build.
actionID := packageAction{
ImportPath: pkg.ImportPath,
CGoVersion: cgo.Version,
CompilerVersion: compiler.Version,
InterpVersion: interp.Version,
CompilerBuildID: string(compilerBuildID),
TinyGoVersion: goenv.Version,
LLVMVersion: llvm.Version,
Config: compilerConfig,
CFlags: pkg.CFlags,
Expand Down
92 changes: 92 additions & 0 deletions builder/buildid.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
package builder

import (
"bytes"
"debug/elf"
"debug/macho"
"encoding/binary"
"fmt"
"io"
"os"
"runtime"
)

// ReadBuildID reads the build ID from the currently running executable.
func ReadBuildID() ([]byte, error) {
executable, err := os.Executable()
if err != nil {
return nil, err
}
f, err := os.Open(executable)
if err != nil {
return nil, err
}
defer f.Close()

switch runtime.GOOS {
case "linux", "freebsd":
// Read the GNU build id section. (Not sure about FreeBSD though...)
file, err := elf.NewFile(f)
if err != nil {
return nil, err
}
for _, section := range file.Sections {
if section.Type != elf.SHT_NOTE || section.Name != ".note.gnu.build-id" {
continue
}
buf := make([]byte, section.Size)
n, err := section.ReadAt(buf, 0)
if uint64(n) != section.Size || err != nil {
return nil, fmt.Errorf("could not read build id: %w", err)
}
return buf, nil
}
case "darwin":
// Read the LC_UUID load command, which contains the equivalent of a
// build ID.
file, err := macho.NewFile(f)
if err != nil {
return nil, err
}
for _, load := range file.Loads {
// Unfortunately, the debug/macho package doesn't support the
// LC_UUID command directly. So we have to read it from
// macho.LoadBytes.
load, ok := load.(macho.LoadBytes)
if !ok {
continue
}
raw := load.Raw()
command := binary.LittleEndian.Uint32(raw)
if command != 0x1b {
// Looking for the LC_UUID load command.
// LC_UUID is defined here as 0x1b:
// https://opensource.apple.com/source/xnu/xnu-4570.71.2/EXTERNAL_HEADERS/mach-o/loader.h.auto.html
continue
}
return raw[4:], nil
}
default:
// On other platforms (such as Windows) there isn't such a convenient
// build ID. Luckily, Go does have an equivalent of the build ID, which
// is stored as a special symbol named go.buildid. You can read it
// using `go tool buildid`, but the code below extracts it directly
// from the binary.
// Unfortunately, because of stripping with the -w flag, no symbol
// table might be available. Therefore, we have to scan the binary
// directly. Luckily the build ID is always at the start of the file.
// For details, see:
// https://github.com/golang/go/blob/master/src/cmd/internal/buildid/buildid.go
fileStart := make([]byte, 4096)
_, err := io.ReadFull(f, fileStart)
index := bytes.Index(fileStart, []byte("\xff Go build ID: \""))
if index < 0 || index > len(fileStart)-103 {
return nil, fmt.Errorf("could not find build id in %s", err)
}
buf := fileStart[index : index+103]
if bytes.HasPrefix(buf, []byte("\xff Go build ID: \"")) && bytes.HasSuffix(buf, []byte("\"\n \xff")) {
return buf[len("\xff Go build ID: \"") : len(buf)-1], nil
}
}
return nil, fmt.Errorf("could not find build ID in %s", executable)
}
6 changes: 0 additions & 6 deletions cgo/cgo.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,6 @@ import (
"golang.org/x/tools/go/ast/astutil"
)

// Version of the cgo package. It must be incremented whenever the cgo package
// is changed in a way that affects the output so that cached package builds
// will be invalidated.
// This version is independent of the TinyGo version number.
const Version = 1 // last change: run libclang once per Go file

// cgoPackage holds all CGo-related information of a package.
type cgoPackage struct {
generated *ast.File
Expand Down
5 changes: 0 additions & 5 deletions compiler/compiler.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,6 @@ import (
"tinygo.org/x/go-llvm"
)

// Version of the compiler pacakge. Must be incremented each time the compiler
// package changes in a way that affects the generated LLVM module.
// This version is independent of the TinyGo version number.
const Version = 25 // last change: add "target-cpu" and "target-features" attributes

func init() {
llvm.InitializeAllTargets()
llvm.InitializeAllTargetMCs()
Expand Down
6 changes: 0 additions & 6 deletions interp/interp.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,6 @@ import (
"tinygo.org/x/go-llvm"
)

// Version of the interp package. It must be incremented whenever the interp
// package is changed in a way that affects the output so that cached package
// builds will be invalidated.
// This version is independent of the TinyGo version number.
const Version = 2 // last change: fix GEP on untyped pointers

// Enable extra checks, which should be disabled by default.
// This may help track down bugs by adding a few more sanity checks.
const checks = true
Expand Down

0 comments on commit 3e109fc

Please sign in to comment.