Add support for Go assembly

Makefile

@@ -118,7 +118,7 @@ endif
 clean:
 	@rm -rf build
 
-FMT_PATHS = ./*.go builder cgo compiler interp loader src/device/arm src/examples src/machine src/os src/reflect src/runtime src/sync src/syscall src/testing src/internal/reflectlite transform
+FMT_PATHS = ./*.go builder cgo compiler goobj interp loader src/device/arm src/examples src/machine src/os src/reflect src/runtime src/sync src/syscall src/testing src/internal/reflectlite transform
 fmt:
 	@gofmt -l -w $(FMT_PATHS)
 fmt-check:

builder/build.go

@@ -22,11 +22,13 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"sync"
 
 	"github.com/gofrs/flock"
 	"github.com/tinygo-org/tinygo/compileopts"
 	"github.com/tinygo-org/tinygo/compiler"
 	"github.com/tinygo-org/tinygo/goenv"
+	"github.com/tinygo-org/tinygo/goobj"
 	"github.com/tinygo-org/tinygo/interp"
 	"github.com/tinygo-org/tinygo/loader"
 	"github.com/tinygo-org/tinygo/stacksize"
@@ -78,6 +80,7 @@ type packageAction struct {
 	OptLevel         int               // LLVM optimization level (0-3)
 	SizeLevel        int               // LLVM optimization for size level (0-2)
 	UndefinedGlobals []string          // globals that are left as external globals (no initializer)
+	GoAsmReferences  map[string]string
 }
 
 // Build performs a single package to executable Go build. It takes in a package
@@ -105,6 +108,14 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil
 		defer os.RemoveAll(dir)
 	}
 
+	// Determine the path to use for caching the build output.
+	cacheDir := goenv.Get("GOCACHE")
+	if cacheDir == "off" {
+		// Use temporary build directory instead, effectively disabling the
+		// build cache.
+		cacheDir = dir
+	}
+
 	// Check for a libc dependency.
 	// As a side effect, this also creates the headers for the given libc, if
 	// the libc needs them.
@@ -193,76 +204,125 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil
 	// Add jobs to compile each package.
 	// Packages that have a cache hit will not be compiled again.
 	var packageJobs []*compileJob
-	packageBitcodePaths := make(map[string]string)
-	packageActionIDs := make(map[string]string)
+	packageActionIDJobs := make(map[string]*compileJob)
+	var linkerDependencies []*compileJob
 	for _, pkg := range lprogram.Sorted() {
 		pkg := pkg // necessary to avoid a race condition
 
+		// Determine which globals should be left undefined in this package.
 		var undefinedGlobals []string
 		for name := range config.Options.GlobalValues[pkg.Pkg.Path()] {
 			undefinedGlobals = append(undefinedGlobals, name)
 		}
 		sort.Strings(undefinedGlobals)
 
-		// Create a cache key: a hash from the action ID below that contains all
-		// the parameters for the build.
-		actionID := packageAction{
-			ImportPath:       pkg.ImportPath,
-			CompilerBuildID:  string(compilerBuildID),
-			TinyGoVersion:    goenv.Version,
-			LLVMVersion:      llvm.Version,
-			Config:           compilerConfig,
-			CFlags:           pkg.CFlags,
-			FileHashes:       make(map[string]string, len(pkg.FileHashes)),
-			Imports:          make(map[string]string, len(pkg.Pkg.Imports())),
-			OptLevel:         optLevel,
-			SizeLevel:        sizeLevel,
-			UndefinedGlobals: undefinedGlobals,
-		}
-		for filePath, hash := range pkg.FileHashes {
-			actionID.FileHashes[filePath] = hex.EncodeToString(hash)
-		}
+		// Create a slice of the action ID jobs for each of the imported
+		var importedPackages []*compileJob
 		for _, imported := range pkg.Pkg.Imports() {
-			hash, ok := packageActionIDs[imported.Path()]
+			job, ok := packageActionIDJobs[imported.Path()]
 			if !ok {
 				return fmt.Errorf("package %s imports %s but couldn't find dependency", pkg.ImportPath, imported.Path())
 			}
-			actionID.Imports[imported.Path()] = hash
+			importedPackages = append(importedPackages, job)
 		}
-		buf, err := json.Marshal(actionID)
-		if err != nil {
-			panic(err) // shouldn't happen
+
+		// References from Go code to assembly functions implemented in Go
+		// assembly. Example: {"math.Sqrt": "__GoABI0_math.Sqrt"}
+		goAsmReferences := map[string]string{}
+		var goAsmReferencesLock sync.Mutex
+
+		// Create jobs to compile all assembly files for this package.
+		compilerDependencies := importedPackages
+		if goobj.SupportsTarget(config.GOOS(), config.GOARCH()) {
+			for _, filename := range pkg.SFiles {
+				abspath := filepath.Join(pkg.Dir, filename)
+				job := &compileJob{
+					description: "compile Go assembly file " + abspath,
+					run: func(job *compileJob) error {
+						// Compile the assembly file using the assembler from the Go
+						// toolchain.
+						result, references, err := compileAsmFile(abspath, dir, pkg.Pkg.Path(), config)
+						job.result = result
+						if err != nil {
+							return err
+						}
+
+						// Add references (both defined and undefined) to the
+						// goAsmReferences map so that the compiler can create
+						// wrapper functions.
+						goAsmReferencesLock.Lock()
+						for internal, external := range references {
+							goAsmReferences[internal] = external
+						}
+						goAsmReferencesLock.Unlock()
+
+						return nil
+					},
+				}
+				compilerDependencies = append(compilerDependencies, job)
+				linkerDependencies = append(linkerDependencies, job)
+			}
 		}
-		hash := sha512.Sum512_224(buf)
-		packageActionIDs[pkg.ImportPath] = hex.EncodeToString(hash[:])
-
-		// Determine the path of the bitcode file (which is a serialized version
-		// of a LLVM module).
-		cacheDir := goenv.Get("GOCACHE")
-		if cacheDir == "off" {
-			// Use temporary build directory instead, effectively disabling the
-			// build cache.
-			cacheDir = dir
+
+		// Create a job that will calculate the action ID for a package compile
+		// job. The action ID is the cache key that is used for caching this
+		// package.
+		packageActionIDJob := &compileJob{
+			description:  "calculate cache key for package " + pkg.ImportPath,
+			dependencies: compilerDependencies,
+			run: func(job *compileJob) error {
+				// Create a cache key: a hash from the action ID below that contains all
+				// the parameters for the build.
+				actionID := packageAction{
+					ImportPath:       pkg.ImportPath,
+					CompilerBuildID:  string(compilerBuildID),
+					TinyGoVersion:    goenv.Version,
+					LLVMVersion:      llvm.Version,
+					Config:           compilerConfig,
+					CFlags:           pkg.CFlags,
+					FileHashes:       make(map[string]string, len(pkg.FileHashes)),
+					Imports:          make(map[string]string, len(pkg.Pkg.Imports())),
+					OptLevel:         optLevel,
+					SizeLevel:        sizeLevel,
+					UndefinedGlobals: undefinedGlobals,
+					GoAsmReferences:  goAsmReferences,
+				}
+				for filePath, hash := range pkg.FileHashes {
+					actionID.FileHashes[filePath] = hex.EncodeToString(hash)
+				}
+				for i, imported := range pkg.Pkg.Imports() {
+					actionID.Imports[imported.Path()] = importedPackages[i].result
+				}
+				buf, err := json.Marshal(actionID)
+				if err != nil {
+					panic(err) // shouldn't happen
+				}
+				hash := sha512.Sum512_224(buf)
+				job.result = hex.EncodeToString(hash[:])
+				return nil
+			},
 		}
-		bitcodePath := filepath.Join(cacheDir, "pkg-"+hex.EncodeToString(hash[:])+".bc")
-		packageBitcodePaths[pkg.ImportPath] = bitcodePath
+		packageActionIDJobs[pkg.ImportPath] = packageActionIDJob
 
-		// The package has not yet been compiled, so create a job to do so.
+		// Now create the job to actually build the package. It will exit early
+		// if the package is already compiled.
 		job := &compileJob{
-			description: "compile package " + pkg.ImportPath,
-			run: func(*compileJob) error {
+			description:  "compile package " + pkg.ImportPath,
+			dependencies: []*compileJob{packageActionIDJob},
+			run: func(job *compileJob) error {
+				job.result = filepath.Join(cacheDir, "pkg-"+job.dependencies[0].result+".bc")
 				// Acquire a lock (if supported).
-				unlock := lock(bitcodePath + ".lock")
+				unlock := lock(job.result + ".lock")
 				defer unlock()
 
-				if _, err := os.Stat(bitcodePath); err == nil {
+				if _, err := os.Stat(job.result); err == nil {
 					// Already cached, don't recreate this package.
 					return nil
 				}
 
 				// Compile AST to IR. The compiler.CompilePackage function will
 				// build the SSA as needed.
-				mod, errs := compiler.CompilePackage(pkg.ImportPath, pkg, program.Package(pkg.Pkg), machine, compilerConfig, config.DumpSSA())
+				mod, errs := compiler.CompilePackage(pkg.ImportPath, pkg, program.Package(pkg.Pkg), machine, compilerConfig, goAsmReferences, config.DumpSSA())
 				if errs != nil {
 					return newMultiError(errs)
 				}
@@ -374,7 +434,7 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil
 				// Write to a temporary path that is renamed to the destination
 				// file to avoid race conditions with other TinyGo invocatiosn
 				// that might also be compiling this package at the same time.
-				f, err := ioutil.TempFile(filepath.Dir(bitcodePath), filepath.Base(bitcodePath))
+				f, err := ioutil.TempFile(filepath.Dir(job.result), filepath.Base(job.result))
 				if err != nil {
 					return err
 				}
@@ -394,13 +454,13 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil
 				if err != nil {
 					// WriteBitcodeToFile doesn't produce a useful error on its
 					// own, so create a somewhat useful error message here.
-					return fmt.Errorf("failed to write bitcode for package %s to file %s", pkg.ImportPath, bitcodePath)
+					return fmt.Errorf("failed to write bitcode for package %s to file %s", pkg.ImportPath, job.result)
 				}
 				err = f.Close()
 				if err != nil {
 					return err
 				}
-				return os.Rename(f.Name(), bitcodePath)
+				return os.Rename(f.Name(), job.result)
 			},
 		}
 		packageJobs = append(packageJobs, job)
@@ -412,13 +472,13 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil
 	programJob := &compileJob{
 		description:  "link+optimize packages (LTO)",
 		dependencies: packageJobs,
-		run: func(*compileJob) error {
+		run: func(job *compileJob) error {
 			// Load and link all the bitcode files. This does not yet optimize
 			// anything, it only links the bitcode files together.
 			ctx := llvm.NewContext()
 			mod = ctx.NewModule("")
-			for _, pkg := range lprogram.Sorted() {
-				pkgMod, err := ctx.ParseBitcodeFile(packageBitcodePaths[pkg.ImportPath])
+			for _, pkgJob := range packageJobs {
+				pkgMod, err := ctx.ParseBitcodeFile(pkgJob.result)
 				if err != nil {
 					return fmt.Errorf("failed to load bitcode file: %w", err)
 				}
@@ -542,7 +602,7 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil
 	}
 
 	// Prepare link command.
-	linkerDependencies := []*compileJob{outputObjectFileJob}
+	linkerDependencies = append(linkerDependencies, outputObjectFileJob)
 	executable := filepath.Join(dir, "main")
 	if config.GOOS() == "windows" {
 		executable += ".exe"
@@ -825,6 +885,62 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil
 	})
 }
 
+// compileAsmFile compiles the given Go assembly file to an ELF object file. The
+// tmpdir is a temporary directory used for storing intermediary files. The
+// importPath is the Go import path for the package the assembly file belongs
+// to.
+func compileAsmFile(path, tmpdir string, importPath string, config *compileopts.Config) (string, map[string]string, error) {
+	// Assemble the Go assembly file. The output is the special Go object
+	// format.
+	goobjfile, err := ioutil.TempFile(tmpdir, "goasm-*.o")
+	if err != nil {
+		return "", nil, err
+	}
+	goobjfile.Close()
+	commandName := filepath.Join(goenv.Get("GOROOT"), "bin", "go")
+	args := []string{"tool", "asm", "-p", importPath, "-o", goobjfile.Name(), "-I", filepath.Join(goenv.Get("GOROOT"), "pkg", "include"), path}
+	cmd := exec.Command(commandName, args...)
+	if config.Options.PrintCommands != nil {
+		config.Options.PrintCommands(commandName, args...)
+	}
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	cmd.Env = append(cmd.Env, "GOOS="+config.GOOS(), "GOARCH="+config.GOARCH())
+	err = cmd.Run()
+	if err != nil {
+		return "", nil, fmt.Errorf("could not invoke Go assembler: %w", err)
+	}
+
+	// Read the Go object file generated by the assembler.
+	buf, err := ioutil.ReadFile(goobjfile.Name())
+	if err != nil {
+		return "", nil, err
+	}
+	obj, err := goobj.ReadGoObj(buf)
+	if err != nil {
+		return "", nil, err
+	}
+
+	// Create relocatable ELF file, for use in linking.
+	data, err := obj.CreateELF()
+	if err != nil {
+		return "", nil, err
+	}
+	elffile, err := ioutil.TempFile(tmpdir, "goasm-*.elf.o")
+	if err != nil {
+		return "", nil, err
+	}
+	_, err = elffile.Write(data)
+	if err != nil {
+		return "", nil, err
+	}
+	err = elffile.Close()
+	if err != nil {
+		return "", nil, err
+	}
+	return elffile.Name(), obj.References(), nil
+}
+
 // optimizeProgram runs a series of optimizations and transformations that are
 // needed to convert a program to its final form. Some transformations are not
 // optional and must be run as the compiler expects them to run.