exec: pull emulator if no local binary found

Dockerfile

@@ -101,8 +101,8 @@ RUN --mount=target=. --mount=target=/root/.cache,type=cache \
 
 FROM scratch AS binaries-linux-helper
 COPY --from=runc /usr/bin/runc /buildkit-runc
-# built from https://github.com/tonistiigi/binfmt/tree/85394e2a1bf0ac9e6c291945e869322bea969445/binfmt
-COPY --from=tonistiigi/binfmt:buildkit / /
+# built from https://github.com/tonistiigi/binfmt/runs/848598596
+COPY --from=tonistiigi/binfmt:buildkit-master@sha256:15b3561ce399a84f27ddd83b82e10c26c31ebab0b19a49aee584498963cba7af /buildkit-* /
 FROM binaries-linux-helper AS binaries-linux
 COPY --from=buildctl /usr/bin/buildctl /
 COPY --from=buildkitd /usr/bin/buildkitd /

solver/llbsolver/ops/exec.go

@@ -32,6 +32,7 @@ import (
 	"github.com/moby/buildkit/solver"
 	"github.com/moby/buildkit/solver/llbsolver"
 	"github.com/moby/buildkit/solver/pb"
+	"github.com/moby/buildkit/source"
 	"github.com/moby/buildkit/util/grpcerrors"
 	"github.com/moby/buildkit/util/progress/logs"
 	utilsystem "github.com/moby/buildkit/util/system"
@@ -47,33 +48,35 @@ import (
 const execCacheType = "buildkit.exec.v0"
 
 type execOp struct {
-	op        *pb.ExecOp
-	cm        cache.Manager
-	sm        *session.Manager
-	md        *metadata.Store
-	exec      executor.Executor
-	w         worker.Worker
-	platform  *pb.Platform
-	numInputs int
+	op            *pb.ExecOp
+	cm            cache.Manager
+	sm            *session.Manager
+	md            *metadata.Store
+	exec          executor.Executor
+	w             worker.Worker
+	sourceManager *source.Manager
+	platform      *pb.Platform
+	numInputs     int
 
 	cacheMounts   map[string]*cacheRefShare
 	cacheMountsMu sync.Mutex
 }
 
-func NewExecOp(v solver.Vertex, op *pb.Op_Exec, platform *pb.Platform, cm cache.Manager, sm *session.Manager, md *metadata.Store, exec executor.Executor, w worker.Worker) (solver.Op, error) {
+func NewExecOp(v solver.Vertex, op *pb.Op_Exec, platform *pb.Platform, cm cache.Manager, m *source.Manager, sm *session.Manager, md *metadata.Store, exec executor.Executor, w worker.Worker) (solver.Op, error) {
 	if err := llbsolver.ValidateOp(&pb.Op{Op: op}); err != nil {
 		return nil, err
 	}
 	return &execOp{
-		op:          op.Exec,
-		cm:          cm,
-		sm:          sm,
-		md:          md,
-		exec:        exec,
-		numInputs:   len(v.Inputs()),
-		w:           w,
-		platform:    platform,
-		cacheMounts: map[string]*cacheRefShare{},
+		op:            op.Exec,
+		cm:            cm,
+		sm:            sm,
+		md:            md,
+		exec:          exec,
+		numInputs:     len(v.Inputs()),
+		w:             w,
+		platform:      platform,
+		cacheMounts:   map[string]*cacheRefShare{},
+		sourceManager: m,
 	}, nil
 }
 
@@ -708,7 +711,7 @@ func (e *execOp) Exec(ctx context.Context, inputs []solver.Result) ([]solver.Res
 		return nil, err
 	}
 
-	emu, err := getEmulator(e.platform, e.cm.IdentityMapping())
+	emu, err := getEmulator(ctx, e.sourceManager, e.w.ContentStore(), e.sm, e.platform, e.cm.IdentityMapping())
 	if err == nil && emu != nil {
 		e.op.Meta.Args = append([]string{qemuMountName}, e.op.Meta.Args...)
 
@@ -717,6 +720,7 @@ func (e *execOp) Exec(ctx context.Context, inputs []solver.Result) ([]solver.Res
 			Src:      emu,
 			Dest:     qemuMountName,
 		})
+		defer emu.Release(context.TODO())
 	}
 	if err != nil {
 		logrus.Warn(err.Error()) // TODO: remove this with pull support

solver/llbsolver/ops/exec_binfmt.go

@@ -2,23 +2,33 @@ package ops
 
 import (
 	"context"
+	"fmt"
 	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"time"
 
+	"github.com/containerd/containerd/content"
 	"github.com/containerd/containerd/mount"
 	"github.com/containerd/containerd/platforms"
+	"github.com/containerd/containerd/reference"
 	"github.com/docker/docker/pkg/idtools"
+	"github.com/moby/buildkit/cache"
+	"github.com/moby/buildkit/client"
+	"github.com/moby/buildkit/session"
 	"github.com/moby/buildkit/snapshot"
 	"github.com/moby/buildkit/solver/pb"
+	"github.com/moby/buildkit/source"
 	"github.com/moby/buildkit/util/binfmt_misc"
+	"github.com/moby/buildkit/util/progress"
 	specs "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 	copy "github.com/tonistiigi/fsutil/copy"
 )
 
 const qemuMountName = "/dev/.buildkit_qemu_emulator"
+const emulatorImage = "docker.io/tonistiigi/binfmt:buildkit@sha256:15b3561ce399a84f27ddd83b82e10c26c31ebab0b19a49aee584498963cba7af"
 
 var qemuArchMap = map[string]string{
 	"arm64":   "aarch64",
@@ -30,12 +40,50 @@ var qemuArchMap = map[string]string{
 }
 
 type emulator struct {
-	path  string
-	idmap *idtools.IdentityMapping
+	mount   snapshot.Mountable
+	release func(context.Context) error
 }
 
 func (e *emulator) Mount(ctx context.Context, readonly bool) (snapshot.Mountable, error) {
-	return &staticEmulatorMount{path: e.path, idmap: e.idmap}, nil
+	return e.mount, nil
+}
+
+func (e *emulator) Release(ctx context.Context) error {
+	if e.release != nil {
+		return e.release(ctx)
+	}
+	return nil
+}
+
+type refEmulatorMount struct {
+	ref  cache.ImmutableRef
+	name string
+}
+
+func (m *refEmulatorMount) Mount() ([]mount.Mount, func() error, error) {
+	mountable, err := m.ref.Mount(context.TODO(), true)
+	if err != nil {
+		return nil, nil, err
+	}
+	mounter := snapshot.LocalMounter(mountable)
+	release := func() error {
+		return mounter.Unmount()
+	}
+	target, err := mounter.Mount()
+	if err != nil {
+		release()
+		return nil, nil, err
+	}
+
+	return []mount.Mount{{
+		Type:    "bind",
+		Source:  filepath.Join(target, "buildkit-qemu-"+m.name),
+		Options: []string{"ro", "bind"},
+	}}, release, nil
+}
+
+func (m *refEmulatorMount) IdentityMapping() *idtools.IdentityMapping {
+	return m.ref.IdentityMapping()
 }
 
 type staticEmulatorMount struct {
@@ -82,7 +130,7 @@ func (m *staticEmulatorMount) IdentityMapping() *idtools.IdentityMapping {
 	return m.idmap
 }
 
-func getEmulator(p *pb.Platform, idmap *idtools.IdentityMapping) (*emulator, error) {
+func getEmulator(ctx context.Context, src *source.Manager, cs content.Store, sm *session.Manager, p *pb.Platform, idmap *idtools.IdentityMapping) (*emulator, error) {
 	all := binfmt_misc.SupportedPlatforms(false)
 	m := make(map[string]struct{}, len(all))
 
@@ -106,9 +154,73 @@ func getEmulator(p *pb.Platform, idmap *idtools.IdentityMapping) (*emulator, err
 	}
 
 	fn, err := exec.LookPath("buildkit-qemu-" + a)
+	if err == nil {
+		return &emulator{mount: &staticEmulatorMount{path: fn, idmap: idmap}}, nil
+	}
+
+	return pullEmulator(ctx, src, cs, sm, pp, a)
+}
+
+func pullEmulator(ctx context.Context, src *source.Manager, cs content.Store, sm *session.Manager, p specs.Platform, name string) (_ *emulator, err error) {
+	id, err := source.NewImageIdentifier(emulatorImage)
+	if err != nil {
+		return nil, err
+	}
+	s := platforms.DefaultSpec()
+	id.Platform = &s
+	id.RecordType = client.UsageRecordTypeInternal
+
+	spec, err := reference.Parse(emulatorImage)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+	var exists bool
+	if dgst := spec.Digest(); dgst != "" {
+		if _, err := cs.Info(ctx, dgst); err == nil {
+			exists = true
+		}
+	}
+	if !exists {
+		defer oneOffProgress(ctx, fmt.Sprintf("pulling emulator for %s", platforms.Format(p)))(err)
+	}
+
+	ctx = progress.WithProgress(ctx, &discard{})
+
+	inst, err := src.Resolve(ctx, id, sm)
 	if err != nil {
-		return nil, errors.Errorf("no emulator available for %v", pp.OS)
+		return nil, err
 	}
 
-	return &emulator{path: fn}, nil
+	ref, err := inst.Snapshot(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	return &emulator{mount: &refEmulatorMount{ref: ref, name: name}, release: ref.Release}, nil
+}
+
+func oneOffProgress(ctx context.Context, id string) func(err error) error {
+	pw, _, _ := progress.FromContext(ctx)
+	now := time.Now()
+	st := progress.Status{
+		Started: &now,
+	}
+	pw.Write(id, st)
+	return func(err error) error {
+		now := time.Now()
+		st.Completed = &now
+		pw.Write(id, st)
+		pw.Close()
+		return err
+	}
+}
+
+type discard struct {
+}
+
+func (d *discard) Write(id string, value interface{}) error {
+	return nil
+}
+func (d *discard) Close() error {
+	return nil
 }

util/pull/resolver.go

@@ -2,6 +2,7 @@ package pull
 
 import (
 	"context"
+	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -34,6 +35,9 @@ func NewResolver(ctx context.Context, hosts docker.RegistryHosts, sm *session.Ma
 }
 
 func EnsureManifestRequested(ctx context.Context, res remotes.Resolver, ref string) {
+	if strings.HasPrefix(ref, "docker.io/") {
+		return
+	}
 	rr := res
 	lr, ok := res.(withLocalResolver)
 	if ok {

worker/base/worker.go

@@ -235,7 +235,7 @@ func (w *Worker) ResolveOp(v solver.Vertex, s frontend.FrontendLLBBridge, sm *se
 		case *pb.Op_Source:
 			return ops.NewSourceOp(v, op, baseOp.Platform, w.SourceManager, sm, w)
 		case *pb.Op_Exec:
-			return ops.NewExecOp(v, op, baseOp.Platform, w.CacheManager, sm, w.MetadataStore, w.Executor, w)
+			return ops.NewExecOp(v, op, baseOp.Platform, w.CacheManager, w.SourceManager, sm, w.MetadataStore, w.Executor, w)
 		case *pb.Op_File:
 			return ops.NewFileOp(v, op, w.CacheManager, w.MetadataStore, w)
 		case *pb.Op_Build: