Implement incremental file sync using client session

Also exposes shared cache and garbage collection/prune
for the source data.

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>

api/server/backend/build/backend.go

@@ -4,11 +4,11 @@ import (
  "fmt"
 
  "github.com/docker/distribution/reference"
+ "github.com/docker/docker/api/types"
  "github.com/docker/docker/api/types/backend"
  "github.com/docker/docker/builder"
- "github.com/docker/docker/builder/dockerfile"
+ "github.com/docker/docker/builder/fscache"
  "github.com/docker/docker/image"
- "github.com/docker/docker/pkg/idtools"
  "github.com/docker/docker/pkg/stringid"
  "github.com/pkg/errors"
  "golang.org/x/net/context"
@@ -20,16 +20,21 @@ type ImageComponent interface {
  TagImageWithReference(image.ID, string, reference.Named) error
 }
 
+// Builder defines interface for running a build
+type Builder interface {
+ Build(context.Context, backend.BuildConfig) (*builder.Result, error)
+}
+
 // Backend provides build functionality to the API router
 type Backend struct {
- manager *dockerfile.BuildManager
+ builder Builder
+ fsCache *fscache.FSCache
  imageComponent ImageComponent
 }
 
 // NewBackend creates a new build backend from components
-func NewBackend(components ImageComponent, builderBackend builder.Backend, sg dockerfile.SessionGetter, idMappings *idtools.IDMappings) (*Backend, error) {
- manager := dockerfile.NewBuildManager(builderBackend, sg, idMappings)
- return &Backend{imageComponent: components, manager: manager}, nil
+func NewBackend(components ImageComponent, builder Builder, fsCache *fscache.FSCache) (*Backend, error) {
+ return &Backend{imageComponent: components, builder: builder, fsCache: fsCache}, nil
 }
 
 // Build builds an image from a Source
@@ -40,7 +45,7 @@ func (b *Backend) Build(ctx context.Context, config backend.BuildConfig) (string
  return "", err
  }
 
- build, err := b.manager.Build(ctx, config)
+ build, err := b.builder.Build(ctx, config)
  if err != nil {
  return "", err
  }
@@ -58,6 +63,15 @@ func (b *Backend) Build(ctx context.Context, config backend.BuildConfig) (string
  return imageID, err
 }
 
+// PruneCache removes all cached build sources
+func (b *Backend) PruneCache(ctx context.Context) (*types.BuildCachePruneReport, error) {
+ size, err := b.fsCache.Prune()
+ if err != nil {
+ return nil, errors.Wrap(err, "failed to prune build cache")
+ }
+ return &types.BuildCachePruneReport{SpaceReclaimed: size}, nil
+}
+
 func squashBuild(build *builder.Result, imageComponent ImageComponent) (string, error) {
  var fromID string
  if build.FromImage != nil {

api/server/router/build/backend.go

@@ -1,6 +1,7 @@
 package build
 
 import (
+ "github.com/docker/docker/api/types"
  "github.com/docker/docker/api/types/backend"
  "golang.org/x/net/context"
 )
@@ -10,6 +11,9 @@ type Backend interface {
  // Build a Docker image returning the id of the image
  // TODO: make this return a reference instead of string
  Build(context.Context, backend.BuildConfig) (string, error)
+
+ // Prune build cache
+ PruneCache(context.Context) (*types.BuildCachePruneReport, error)
 }
 
 type experimentalProvider interface {

api/server/router/build/build.go

@@ -24,5 +24,6 @@ func (r *buildRouter) Routes() []router.Route {
 func (r *buildRouter) initRoutes() {
  r.routes = []router.Route{
  router.NewPostRoute("/build", r.postBuild, router.WithCancel),
+ router.NewPostRoute("/build/prune", r.postPrune, router.WithCancel),
  }
 }

api/server/router/build/build_routes.go

@@ -132,6 +132,14 @@ func newImageBuildOptions(ctx context.Context, r *http.Request) (*types.ImageBui
  return options, nil
 }
 
+func (br *buildRouter) postPrune(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
+ report, err := br.backend.PruneCache(ctx)
+ if err != nil {
+ return err
+ }
+ return httputils.WriteJSON(w, http.StatusOK, report)
+}
+
 func (br *buildRouter) postBuild(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
  var (
  notVerboseBuffer = bytes.NewBuffer(nil)

api/server/router/system/system.go

@@ -2,6 +2,7 @@ package system
 
 import (
  "github.com/docker/docker/api/server/router"
+ "github.com/docker/docker/builder/fscache"
  "github.com/docker/docker/daemon/cluster"
 )
 
@@ -11,13 +12,15 @@ type systemRouter struct {
  backend Backend
  cluster *cluster.Cluster
  routes []router.Route
+ builder *fscache.FSCache
 }
 
 // NewRouter initializes a new system router
-func NewRouter(b Backend, c *cluster.Cluster) router.Router {
+func NewRouter(b Backend, c *cluster.Cluster, fscache *fscache.FSCache) router.Router {
  r := &systemRouter{
  backend: b,
  cluster: c,
+ builder: fscache,
  }
 
  r.routes = []router.Route{

api/server/router/system/system_routes.go

@@ -17,6 +17,7 @@ import (
  timetypes "github.com/docker/docker/api/types/time"
  "github.com/docker/docker/api/types/versions"
  "github.com/docker/docker/pkg/ioutils"
+ pkgerrors "github.com/pkg/errors"
  "golang.org/x/net/context"
 )
 
@@ -75,6 +76,11 @@ func (s *systemRouter) getDiskUsage(ctx context.Context, w http.ResponseWriter,
  if err != nil {
  return err
  }
+ builderSize, err := s.builder.DiskUsage()
+ if err != nil {
+ return pkgerrors.Wrap(err, "error getting build cache usage")
+ }
+ du.BuilderSize = builderSize
 
  return httputils.WriteJSON(w, http.StatusOK, du)
 }

api/swagger.yaml

@@ -4745,6 +4745,27 @@ paths:
  schema:
  $ref: "#/definitions/ErrorResponse"
  tags: ["Image"]
+ /build/prune:
+ post:
+ summary: "Delete builder cache"
+ produces:
+ - "application/json"
+ operationId: "BuildPrune"
+ responses:
+ 200:
+ description: "No error"
+ schema:
+ type: "object"
+ properties:
+ SpaceReclaimed:
+ description: "Disk space reclaimed in bytes"
+ type: "integer"
+ format: "int64"
+ 500:
+ description: "Server error"
+ schema:
+ $ref: "#/definitions/ErrorResponse"
+ tags: ["Image"]
  /images/create:
  post:
  summary: "Create an image"

api/types/types.go

@@ -489,10 +489,11 @@ type Runtime struct {
 // DiskUsage contains response of Engine API:
 // GET "/system/df"
 type DiskUsage struct {
- LayersSize int64
- Images []*ImageSummary
- Containers []*Container
- Volumes []*Volume
+ LayersSize int64
+ Images []*ImageSummary
+ Containers []*Container
+ Volumes []*Volume
+ BuilderSize int64
 }
 
 // ContainersPruneReport contains the response for Engine API:
@@ -516,6 +517,12 @@ type ImagesPruneReport struct {
  SpaceReclaimed uint64
 }
 
+// BuildCachePruneReport contains the response for Engine API:
+// POST "/build/prune"
+type BuildCachePruneReport struct {
+ SpaceReclaimed uint64
+}
+
 // NetworksPruneReport contains the response for Engine API:
 // POST "/networks/prune"
 type NetworksPruneReport struct {

builder/dockerfile/builder.go

@@ -7,6 +7,7 @@ import (
  "io/ioutil"
  "runtime"
  "strings"
+ "time"
 
  "github.com/Sirupsen/logrus"
  "github.com/docker/docker/api/types"
@@ -15,6 +16,7 @@ import (
  "github.com/docker/docker/builder"
  "github.com/docker/docker/builder/dockerfile/command"
  "github.com/docker/docker/builder/dockerfile/parser"
+ "github.com/docker/docker/builder/fscache"
  "github.com/docker/docker/builder/remotecontext"
  "github.com/docker/docker/client/session"
  "github.com/docker/docker/pkg/archive"
@@ -52,16 +54,22 @@ type BuildManager struct {
  backend builder.Backend
  pathCache pathCache // TODO: make this persistent
  sg SessionGetter
+ fsCache *fscache.FSCache
 }
 
 // NewBuildManager creates a BuildManager
-func NewBuildManager(b builder.Backend, sg SessionGetter, idMappings *idtools.IDMappings) *BuildManager {
- return &BuildManager{
+func NewBuildManager(b builder.Backend, sg SessionGetter, fsCache *fscache.FSCache, idMappings *idtools.IDMappings) (*BuildManager, error) {
+ bm := &BuildManager{
  backend: b,
  pathCache: &syncmap.Map{},
  sg: sg,
  archiver: chrootarchive.NewArchiver(idMappings),
+ fsCache: fsCache,
  }
+ if err := fsCache.RegisterTransport(remotecontext.ClientSessionRemote, NewClientSessionTransport()); err != nil {
+ return nil, err
+ }
+ return bm, nil
 }
 
 // Build starts a new build from a BuildConfig
@@ -75,13 +83,13 @@ func (bm *BuildManager) Build(ctx context.Context, config backend.BuildConfig) (
  if err != nil {
  return nil, err
  }
- if source != nil {
- defer func() {
+ defer func() {
+ if source != nil {
  if err := source.Close(); err != nil {
  logrus.Debugf("[BUILDER] failed to remove temporary context: %v", err)
  }
- }()
- }
+ }
+ }()
 
  // TODO @jhowardmsft LCOW support - this will require rework to allow both linux and Windows simultaneously.
  // This is an interim solution to hardcode to linux if LCOW is turned on.
@@ -95,8 +103,10 @@ func (bm *BuildManager) Build(ctx context.Context, config backend.BuildConfig) (
  ctx, cancel := context.WithCancel(ctx)
  defer cancel()
 
- if err := bm.initializeClientSession(ctx, cancel, config.Options); err != nil {
+ if src, err := bm.initializeClientSession(ctx, cancel, config.Options); err != nil {
  return nil, err
+ } else if src != nil {
+ source = src
  }
 
  builderOptions := builderOptions{
@@ -111,20 +121,38 @@ func (bm *BuildManager) Build(ctx context.Context, config backend.BuildConfig) (
  return newBuilder(ctx, builderOptions).build(source, dockerfile)
 }
 
-func (bm *BuildManager) initializeClientSession(ctx context.Context, cancel func(), options *types.ImageBuildOptions) error {
+func (bm *BuildManager) initializeClientSession(ctx context.Context, cancel func(), options *types.ImageBuildOptions) (builder.Source, error) {
  if options.SessionID == "" || bm.sg == nil {
- return nil
+ return nil, nil
  }
  logrus.Debug("client is session enabled")
+
+ ctx, cancelCtx := context.WithTimeout(ctx, sessionConnectTimeout)
+ defer cancelCtx()
+
  c, err := bm.sg.Get(ctx, options.SessionID)
  if err != nil {
- return err
+ return nil, err
  }
  go func() {
  <-c.Context().Done()
  cancel()
  }()
- return nil
+ if options.RemoteContext == remotecontext.ClientSessionRemote {
+ st := time.Now()
+ csi, err := NewClientSessionSourceIdentifier(ctx, bm.sg,
+ options.SessionID, []string{"/"})
+ if err != nil {
+ return nil, err
+ }
+ src, err := bm.fsCache.SyncFrom(ctx, csi)
+ if err != nil {
+ return nil, err
+ }
+ logrus.Debugf("sync-time: %v", time.Since(st))
+ return src, nil
+ }
+ return nil, nil
 }
 
 // builderOptions are the dependencies required by the builder

builder/dockerfile/clientsession.go

@@ -0,0 +1,78 @@
+package dockerfile
+
+import (
+ "time"
+
+ "github.com/docker/docker/builder/fscache"
+ "github.com/docker/docker/builder/remotecontext"
+ "github.com/docker/docker/client/session"
+ "github.com/docker/docker/client/session/filesync"
+ "github.com/pkg/errors"
+ "golang.org/x/net/context"
+)
+
+const sessionConnectTimeout = 5 * time.Second
+
+// ClientSessionTransport is a transport for copying files from docker client
+// to the daemon.
+type ClientSessionTransport struct{}
+
+// NewClientSessionTransport returns new ClientSessionTransport instance
+func NewClientSessionTransport() *ClientSessionTransport {
+ return &ClientSessionTransport{}
+}
+
+// Copy data from a remote to a destination directory.
+func (cst *ClientSessionTransport) Copy(ctx context.Context, id fscache.RemoteIdentifier, dest string, cu filesync.CacheUpdater) error {
+ csi, ok := id.(*ClientSessionSourceIdentifier)
+ if !ok {
+ return errors.New("invalid identifier for client session")
+ }
+
+ return filesync.FSSync(ctx, csi.caller, filesync.FSSendRequestOpt{
+ SrcPaths: csi.srcPaths,
+ DestDir: dest,
+ CacheUpdater: cu,
+ })
+}
+
+// ClientSessionSourceIdentifier is an identifier that can be used for requesting
+// files from remote client
+type ClientSessionSourceIdentifier struct {
+ srcPaths []string
+ caller session.Caller
+ sharedKey string
+ uuid string
+}
+
+// NewClientSessionSourceIdentifier returns new ClientSessionSourceIdentifier instance
+func NewClientSessionSourceIdentifier(ctx context.Context, sg SessionGetter, uuid string, sources []string) (*ClientSessionSourceIdentifier, error) {
+ csi := &ClientSessionSourceIdentifier{
+ uuid: uuid,
+ srcPaths: sources,
+ }
+ caller, err := sg.Get(ctx, uuid)
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed to get session for %s", uuid)
+ }
+
+ csi.caller = caller
+ return csi, nil
+}
+
+// Transport returns transport identifier for remote identifier
+func (csi *ClientSessionSourceIdentifier) Transport() string {
+ return remotecontext.ClientSessionRemote
+}
+
+// SharedKey returns shared key for remote identifier. Shared key is used
+// for finding the base for a repeated transfer.
+func (csi *ClientSessionSourceIdentifier) SharedKey() string {
+ return csi.caller.SharedKey()
+}
+
+// Key returns unique key for remote identifier. Requests with same key return
+// same data.
+func (csi *ClientSessionSourceIdentifier) Key() string {
+ return csi.uuid
+}