From 6e9c39a2eb5c5b63d2b75a3d27bd51cb7b97be7f Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Wed, 10 Jun 2020 19:40:07 -0700 Subject: [PATCH 1/2] exec: pull emulator if no local binary found Signed-off-by: Tonis Tiigi --- solver/llbsolver/ops/exec.go | 42 +++++----- solver/llbsolver/ops/exec_binfmt.go | 124 ++++++++++++++++++++++++++-- util/pull/resolver.go | 4 + worker/base/worker.go | 2 +- 4 files changed, 146 insertions(+), 26 deletions(-) diff --git a/solver/llbsolver/ops/exec.go b/solver/llbsolver/ops/exec.go index 54f0c3a85235..e5045753c120 100644 --- a/solver/llbsolver/ops/exec.go +++ b/solver/llbsolver/ops/exec.go @@ -32,6 +32,7 @@ import ( "github.com/moby/buildkit/solver" "github.com/moby/buildkit/solver/llbsolver" "github.com/moby/buildkit/solver/pb" + "github.com/moby/buildkit/source" "github.com/moby/buildkit/util/grpcerrors" "github.com/moby/buildkit/util/progress/logs" utilsystem "github.com/moby/buildkit/util/system" @@ -47,33 +48,35 @@ import ( const execCacheType = "buildkit.exec.v0" type execOp struct { - op *pb.ExecOp - cm cache.Manager - sm *session.Manager - md *metadata.Store - exec executor.Executor - w worker.Worker - platform *pb.Platform - numInputs int + op *pb.ExecOp + cm cache.Manager + sm *session.Manager + md *metadata.Store + exec executor.Executor + w worker.Worker + sourceManager *source.Manager + platform *pb.Platform + numInputs int cacheMounts map[string]*cacheRefShare cacheMountsMu sync.Mutex } -func NewExecOp(v solver.Vertex, op *pb.Op_Exec, platform *pb.Platform, cm cache.Manager, sm *session.Manager, md *metadata.Store, exec executor.Executor, w worker.Worker) (solver.Op, error) { +func NewExecOp(v solver.Vertex, op *pb.Op_Exec, platform *pb.Platform, cm cache.Manager, m *source.Manager, sm *session.Manager, md *metadata.Store, exec executor.Executor, w worker.Worker) (solver.Op, error) { if err := llbsolver.ValidateOp(&pb.Op{Op: op}); err != nil { return nil, err } return &execOp{ - op: op.Exec, - cm: cm, - sm: sm, - md: md, - exec: exec, - numInputs: len(v.Inputs()), - w: w, - platform: platform, - cacheMounts: map[string]*cacheRefShare{}, + op: op.Exec, + cm: cm, + sm: sm, + md: md, + exec: exec, + numInputs: len(v.Inputs()), + w: w, + platform: platform, + cacheMounts: map[string]*cacheRefShare{}, + sourceManager: m, }, nil } @@ -708,7 +711,7 @@ func (e *execOp) Exec(ctx context.Context, inputs []solver.Result) ([]solver.Res return nil, err } - emu, err := getEmulator(e.platform, e.cm.IdentityMapping()) + emu, err := getEmulator(ctx, e.sourceManager, e.w.ContentStore(), e.sm, e.platform, e.cm.IdentityMapping()) if err == nil && emu != nil { e.op.Meta.Args = append([]string{qemuMountName}, e.op.Meta.Args...) @@ -717,6 +720,7 @@ func (e *execOp) Exec(ctx context.Context, inputs []solver.Result) ([]solver.Res Src: emu, Dest: qemuMountName, }) + defer emu.Release(context.TODO()) } if err != nil { logrus.Warn(err.Error()) // TODO: remove this with pull support diff --git a/solver/llbsolver/ops/exec_binfmt.go b/solver/llbsolver/ops/exec_binfmt.go index 5eea8ff95388..9a1388a8c8ba 100644 --- a/solver/llbsolver/ops/exec_binfmt.go +++ b/solver/llbsolver/ops/exec_binfmt.go @@ -2,23 +2,33 @@ package ops import ( "context" + "fmt" "io/ioutil" "os" "os/exec" "path/filepath" + "time" + "github.com/containerd/containerd/content" "github.com/containerd/containerd/mount" "github.com/containerd/containerd/platforms" + "github.com/containerd/containerd/reference" "github.com/docker/docker/pkg/idtools" + "github.com/moby/buildkit/cache" + "github.com/moby/buildkit/client" + "github.com/moby/buildkit/session" "github.com/moby/buildkit/snapshot" "github.com/moby/buildkit/solver/pb" + "github.com/moby/buildkit/source" "github.com/moby/buildkit/util/binfmt_misc" + "github.com/moby/buildkit/util/progress" specs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" copy "github.com/tonistiigi/fsutil/copy" ) const qemuMountName = "/dev/.buildkit_qemu_emulator" +const emulatorImage = "docker.io/tonistiigi/binfmt:buildkit@sha256:5e7df2cf5373ba557ff2e61994cbc4d16adbd0627db1ed55acde3b7a16a693ba" var qemuArchMap = map[string]string{ "arm64": "aarch64", @@ -30,12 +40,50 @@ var qemuArchMap = map[string]string{ } type emulator struct { - path string - idmap *idtools.IdentityMapping + mount snapshot.Mountable + release func(context.Context) error } func (e *emulator) Mount(ctx context.Context, readonly bool) (snapshot.Mountable, error) { - return &staticEmulatorMount{path: e.path, idmap: e.idmap}, nil + return e.mount, nil +} + +func (e *emulator) Release(ctx context.Context) error { + if e.release != nil { + return e.release(ctx) + } + return nil +} + +type refEmulatorMount struct { + ref cache.ImmutableRef + name string +} + +func (m *refEmulatorMount) Mount() ([]mount.Mount, func() error, error) { + mountable, err := m.ref.Mount(context.TODO(), true) + if err != nil { + return nil, nil, err + } + mounter := snapshot.LocalMounter(mountable) + release := func() error { + return mounter.Unmount() + } + target, err := mounter.Mount() + if err != nil { + release() + return nil, nil, err + } + + return []mount.Mount{{ + Type: "bind", + Source: filepath.Join(target, "buildkit-qemu-"+m.name), + Options: []string{"ro", "bind"}, + }}, release, nil +} + +func (m *refEmulatorMount) IdentityMapping() *idtools.IdentityMapping { + return m.ref.IdentityMapping() } type staticEmulatorMount struct { @@ -82,7 +130,7 @@ func (m *staticEmulatorMount) IdentityMapping() *idtools.IdentityMapping { return m.idmap } -func getEmulator(p *pb.Platform, idmap *idtools.IdentityMapping) (*emulator, error) { +func getEmulator(ctx context.Context, src *source.Manager, cs content.Store, sm *session.Manager, p *pb.Platform, idmap *idtools.IdentityMapping) (*emulator, error) { all := binfmt_misc.SupportedPlatforms(false) m := make(map[string]struct{}, len(all)) @@ -106,9 +154,73 @@ func getEmulator(p *pb.Platform, idmap *idtools.IdentityMapping) (*emulator, err } fn, err := exec.LookPath("buildkit-qemu-" + a) + if err == nil { + return &emulator{mount: &staticEmulatorMount{path: fn, idmap: idmap}}, nil + } + + return pullEmulator(ctx, src, cs, sm, pp, a) +} + +func pullEmulator(ctx context.Context, src *source.Manager, cs content.Store, sm *session.Manager, p specs.Platform, name string) (_ *emulator, err error) { + id, err := source.NewImageIdentifier(emulatorImage) + if err != nil { + return nil, err + } + s := platforms.DefaultSpec() + id.Platform = &s + id.RecordType = client.UsageRecordTypeInternal + + spec, err := reference.Parse(emulatorImage) + if err != nil { + return nil, errors.WithStack(err) + } + var exists bool + if dgst := spec.Digest(); dgst != "" { + if _, err := cs.Info(ctx, dgst); err == nil { + exists = true + } + } + if !exists { + defer oneOffProgress(ctx, fmt.Sprintf("pulling emulator for %s", platforms.Format(p)))(err) + } + + ctx = progress.WithProgress(ctx, &discard{}) + + inst, err := src.Resolve(ctx, id, sm) if err != nil { - return nil, errors.Errorf("no emulator available for %v", pp.OS) + return nil, err } - return &emulator{path: fn}, nil + ref, err := inst.Snapshot(ctx) + if err != nil { + return nil, err + } + + return &emulator{mount: &refEmulatorMount{ref: ref, name: name}, release: ref.Release}, nil +} + +func oneOffProgress(ctx context.Context, id string) func(err error) error { + pw, _, _ := progress.FromContext(ctx) + now := time.Now() + st := progress.Status{ + Started: &now, + } + pw.Write(id, st) + return func(err error) error { + now := time.Now() + st.Completed = &now + pw.Write(id, st) + pw.Close() + return err + } +} + +type discard struct { +} + +func (d *discard) Write(id string, value interface{}) error { + return nil +} +func (d *discard) Close() error { + return nil } diff --git a/util/pull/resolver.go b/util/pull/resolver.go index ca425c24a7f2..1afb636368fb 100644 --- a/util/pull/resolver.go +++ b/util/pull/resolver.go @@ -2,6 +2,7 @@ package pull import ( "context" + "strings" "sync" "sync/atomic" "time" @@ -34,6 +35,9 @@ func NewResolver(ctx context.Context, hosts docker.RegistryHosts, sm *session.Ma } func EnsureManifestRequested(ctx context.Context, res remotes.Resolver, ref string) { + if strings.HasPrefix(ref, "docker.io/") { + return + } rr := res lr, ok := res.(withLocalResolver) if ok { diff --git a/worker/base/worker.go b/worker/base/worker.go index 1cbdd6fa0ebb..cbb919177840 100644 --- a/worker/base/worker.go +++ b/worker/base/worker.go @@ -235,7 +235,7 @@ func (w *Worker) ResolveOp(v solver.Vertex, s frontend.FrontendLLBBridge, sm *se case *pb.Op_Source: return ops.NewSourceOp(v, op, baseOp.Platform, w.SourceManager, sm, w) case *pb.Op_Exec: - return ops.NewExecOp(v, op, baseOp.Platform, w.CacheManager, sm, w.MetadataStore, w.Executor, w) + return ops.NewExecOp(v, op, baseOp.Platform, w.CacheManager, w.SourceManager, sm, w.MetadataStore, w.Executor, w) case *pb.Op_File: return ops.NewFileOp(v, op, w.CacheManager, w.MetadataStore, w) case *pb.Op_Build: From 8e2cc0f0992943ca290097b36a538b8097e8d26b Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Tue, 7 Jul 2020 23:39:25 -0700 Subject: [PATCH 2/2] temp: update emulator images Signed-off-by: Tonis Tiigi --- Dockerfile | 4 ++-- solver/llbsolver/ops/exec_binfmt.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index d42bab248465..c98cab0fc054 100644 --- a/Dockerfile +++ b/Dockerfile @@ -101,8 +101,8 @@ RUN --mount=target=. --mount=target=/root/.cache,type=cache \ FROM scratch AS binaries-linux-helper COPY --from=runc /usr/bin/runc /buildkit-runc -# built from https://github.com/tonistiigi/binfmt/tree/85394e2a1bf0ac9e6c291945e869322bea969445/binfmt -COPY --from=tonistiigi/binfmt:buildkit / / +# built from https://github.com/tonistiigi/binfmt/runs/848598596 +COPY --from=tonistiigi/binfmt:buildkit-master@sha256:15b3561ce399a84f27ddd83b82e10c26c31ebab0b19a49aee584498963cba7af /buildkit-* / FROM binaries-linux-helper AS binaries-linux COPY --from=buildctl /usr/bin/buildctl / COPY --from=buildkitd /usr/bin/buildkitd / diff --git a/solver/llbsolver/ops/exec_binfmt.go b/solver/llbsolver/ops/exec_binfmt.go index 9a1388a8c8ba..8eea2112b400 100644 --- a/solver/llbsolver/ops/exec_binfmt.go +++ b/solver/llbsolver/ops/exec_binfmt.go @@ -28,7 +28,7 @@ import ( ) const qemuMountName = "/dev/.buildkit_qemu_emulator" -const emulatorImage = "docker.io/tonistiigi/binfmt:buildkit@sha256:5e7df2cf5373ba557ff2e61994cbc4d16adbd0627db1ed55acde3b7a16a693ba" +const emulatorImage = "docker.io/tonistiigi/binfmt:buildkit@sha256:15b3561ce399a84f27ddd83b82e10c26c31ebab0b19a49aee584498963cba7af" var qemuArchMap = map[string]string{ "arm64": "aarch64",