From 868fefcf4e40cbf00f567f7b8676cb2de85be762 Mon Sep 17 00:00:00 2001 From: Petros Angelatos Date: Fri, 6 Oct 2017 13:30:02 -0700 Subject: [PATCH 1/5] pkg/ioutils: export SeekerSize utility Signed-off-by: Petros Angelatos --- pkg/ioutils/concat.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/ioutils/concat.go b/pkg/ioutils/concat.go index c036ff0c72..9e05af4ef3 100644 --- a/pkg/ioutils/concat.go +++ b/pkg/ioutils/concat.go @@ -19,7 +19,7 @@ func max(x, y int) int { return x } -func seekerSize(s io.Seeker) (int64, error) { +func SeekerSize(s io.Seeker) (int64, error) { cur, err := s.Seek(0, io.SeekCurrent) if err != nil { return 0, err @@ -109,12 +109,12 @@ func (self *concatReadSeekCloser) Close() error { } func ConcatReadSeekClosers(a, b ReadSeekCloser) (ReadSeekCloser, error) { - aSize, err := seekerSize(a) + aSize, err := SeekerSize(a) if err != nil { return nil, err } - bSize, err := seekerSize(b) + bSize, err := SeekerSize(b) if err != nil { return nil, err } From 8e777352ee0f1444674b78eaea95ac576f96d8da Mon Sep 17 00:00:00 2001 From: Petros Angelatos Date: Fri, 6 Oct 2017 13:36:57 -0700 Subject: [PATCH 2/5] image/delta: client support with progress reporting Signed-off-by: Petros Angelatos --- api/server/router/delta/backend.go | 6 ++- api/server/router/delta/delta_routes.go | 21 +++++--- client/interface.go | 1 + daemon/create.go | 66 +++++++++++++++++++------ 4 files changed, 70 insertions(+), 24 deletions(-) diff --git a/api/server/router/delta/backend.go b/api/server/router/delta/backend.go index da91bd09f6..0d6e3cedab 100644 --- a/api/server/router/delta/backend.go +++ b/api/server/router/delta/backend.go @@ -1,7 +1,11 @@ package delta +import ( + "io" +) + // Backend is the methods that need to be implemented to provide // delta specific functionality. type Backend interface { - DeltaCreate(deltaSrc, deltaDest string) (imageID string, err error) + DeltaCreate(deltaSrc, deltaDest string, outStream io.Writer) error } diff --git a/api/server/router/delta/delta_routes.go b/api/server/router/delta/delta_routes.go index d22ecf86d8..848ed60ca4 100644 --- a/api/server/router/delta/delta_routes.go +++ b/api/server/router/delta/delta_routes.go @@ -4,7 +4,8 @@ import ( "net/http" "github.com/docker/docker/api/server/httputils" - "github.com/docker/docker/api/types" + "github.com/docker/docker/pkg/ioutils" + "github.com/docker/docker/pkg/streamformatter" "golang.org/x/net/context" ) @@ -16,12 +17,16 @@ func (d *deltaRouter) postDeltasCreate(ctx context.Context, w http.ResponseWrite deltaSrc := r.Form.Get("src") deltaDest := r.Form.Get("dest") - imgID, err := d.backend.DeltaCreate(deltaSrc, deltaDest) - if err != nil { - return err - } + output := ioutils.NewWriteFlusher(w) + defer output.Close() - return httputils.WriteJSON(w, http.StatusCreated, &types.IDResponse{ - ID: string(imgID), - }) + w.Header().Set("Content-Type", "application/json") + + if err := d.backend.DeltaCreate(deltaSrc, deltaDest, output); err != nil { + if !output.Flushed() { + return err + } + output.Write(streamformatter.FormatError(err)) + } + return nil } diff --git a/client/interface.go b/client/interface.go index acd4de1dbd..1eb338501c 100644 --- a/client/interface.go +++ b/client/interface.go @@ -84,6 +84,7 @@ type ImageAPIClient interface { ImageBuild(ctx context.Context, context io.Reader, options types.ImageBuildOptions) (types.ImageBuildResponse, error) BuildCachePrune(ctx context.Context) (*types.BuildCachePruneReport, error) ImageCreate(ctx context.Context, parentReference string, options types.ImageCreateOptions) (io.ReadCloser, error) + ImageDelta(ctx context.Context, src, dest string) (io.ReadCloser, error) ImageHistory(ctx context.Context, image string) ([]image.HistoryResponseItem, error) ImageImport(ctx context.Context, source types.ImageImportSource, ref string, options types.ImageImportOptions) (io.ReadCloser, error) ImageInspectWithRaw(ctx context.Context, image string) (types.ImageInspect, []byte, error) diff --git a/daemon/create.go b/daemon/create.go index 58869e943b..3190e960b0 100644 --- a/daemon/create.go +++ b/daemon/create.go @@ -25,7 +25,10 @@ import ( "github.com/docker/docker/image" "github.com/docker/docker/layer" "github.com/docker/docker/pkg/idtools" + "github.com/docker/docker/pkg/ioutils" + "github.com/docker/docker/pkg/progress" "github.com/docker/docker/pkg/stringid" + "github.com/docker/docker/pkg/streamformatter" "github.com/docker/docker/pkg/system" "github.com/docker/docker/runconfig" "github.com/opencontainers/go-digest" @@ -341,15 +344,17 @@ func (daemon *Daemon) verifyNetworkingConfig(nwConfig *networktypes.NetworkingCo // DeltaCreate creates a delta of the specified src and dest images // This is called directly from the Engine API -func (daemon *Daemon) DeltaCreate(deltaSrc, deltaDest string) (string, error) { +func (daemon *Daemon) DeltaCreate(deltaSrc, deltaDest string, outStream io.Writer) error { + progressOutput := streamformatter.NewJSONProgressOutput(outStream, false) + srcImg, err := daemon.GetImage(deltaSrc) if err != nil { - return "", errors.Wrapf(err, "no such image: %s", deltaSrc) + return errors.Wrapf(err, "no such image: %s", deltaSrc) } dstImg, err := daemon.GetImage(deltaDest) if err != nil { - return "", errors.Wrapf(err, "no such image: %s", deltaDest) + return errors.Wrapf(err, "no such image: %s", deltaDest) } is := daemon.stores[dstImg.Platform()].imageStore @@ -357,26 +362,42 @@ func (daemon *Daemon) DeltaCreate(deltaSrc, deltaDest string) (string, error) { srcData, err := is.GetTarSeekStream(srcImg.ID()) if err != nil { - return "", err + return err } defer srcData.Close() - srcSig, err := librsync.Signature(bufio.NewReaderSize(srcData, 65536), ioutil.Discard, 512, 32, librsync.BLAKE2_SIG_MAGIC) + srcDataLen, err := ioutils.SeekerSize(srcData) + if err != nil { + return err + } + + progressReader := progress.NewProgressReader(srcData, progressOutput, srcDataLen, deltaSrc, "Fingerprinting") + defer progressReader.Close() + + srcSig, err := librsync.Signature(bufio.NewReaderSize(progressReader, 65536), ioutil.Discard, 512, 32, librsync.BLAKE2_SIG_MAGIC) if err != nil { - return "", err + return err } + progress.Update(progressOutput, deltaSrc, "Fingerprint complete") + deltaRootFS := image.NewRootFS() + for _, diffID := range dstImg.RootFS.DiffIDs { + progress.Update(progressOutput, stringid.TruncateID(diffID.String()), "Waiting") + } + for i, diffID := range dstImg.RootFS.DiffIDs { var ( layerData io.Reader platform layer.Platform ) + commonLayer := false // We're only interested in layers that are different. Push empty // layers for common layers if i < len(srcImg.RootFS.DiffIDs) && srcImg.RootFS.DiffIDs[i] == diffID { + commonLayer = true layerData, _ = layer.EmptyLayer.TarStream() platform = layer.EmptyLayer.Platform() } else { @@ -385,7 +406,7 @@ func (daemon *Daemon) DeltaCreate(deltaSrc, deltaDest string) (string, error) { l, err := ls.Get(dstRootFS.ChainID()) if err != nil { - return "", err + return err } defer layer.ReleaseAndLog(ls, l) @@ -393,23 +414,31 @@ func (daemon *Daemon) DeltaCreate(deltaSrc, deltaDest string) (string, error) { input, err := l.TarStream() if err != nil { - return "", err + return err } defer input.Close() + inputSize, err := l.DiffSize() + if err != nil { + return err + } + + progressReader := progress.NewProgressReader(input, progressOutput, inputSize, stringid.TruncateID(diffID.String()), "Computing delta") + defer progressReader.Close() + pR, pW := io.Pipe() layerData = pR tmpDelta, err := ioutil.TempFile("", "docker-delta-") if err != nil { - return "", err + return err } defer os.Remove(tmpDelta.Name()) go func() { w := bufio.NewWriter(tmpDelta) - err := librsync.Delta(srcSig, bufio.NewReader(input), w) + err := librsync.Delta(srcSig, bufio.NewReader(progressReader), w) if err != nil { pW.CloseWithError(err) return @@ -456,10 +485,16 @@ func (daemon *Daemon) DeltaCreate(deltaSrc, deltaDest string) (string, error) { newLayer, err := ls.Register(layerData, deltaRootFS.ChainID(), platform) if err != nil { - return "", err + return err } defer layer.ReleaseAndLog(ls, newLayer) + if commonLayer { + progress.Update(progressOutput, stringid.TruncateID(diffID.String()), "Skipping common layer") + } else { + progress.Update(progressOutput, stringid.TruncateID(diffID.String()), "Delta complete") + } + deltaRootFS.Append(newLayer.DiffID()) } @@ -478,12 +513,12 @@ func (daemon *Daemon) DeltaCreate(deltaSrc, deltaDest string) (string, error) { rawConfig, err := json.Marshal(config) if err != nil { - return "", err + return err } id, err := is.Create(rawConfig) if err != nil { - return "", err + return err } ref, _ := reference.WithName("delta") @@ -493,8 +528,9 @@ func (daemon *Daemon) DeltaCreate(deltaSrc, deltaDest string) (string, error) { ref2, _ := reference.WithTag(ref, deltaTag) if err := daemon.TagImageWithReference(id, "linux", ref2); err != nil { - return "", err + return err } - return id.String(), nil + outStream.Write(streamformatter.FormatStatus("", id.String())) + return nil } From 02d889d3459c9b376a83384309a353c278eb5f2b Mon Sep 17 00:00:00 2001 From: Petros Angelatos Date: Fri, 6 Oct 2017 14:10:06 -0700 Subject: [PATCH 3/5] api: move delta creation under POST images/delta Signed-off-by: Petros Angelatos --- api/server/router/delta/backend.go | 11 --------- api/server/router/delta/delta.go | 30 ----------------------- api/server/router/delta/delta_routes.go | 32 ------------------------- api/server/router/image/backend.go | 1 + api/server/router/image/image.go | 1 + api/server/router/image/image_routes.go | 22 +++++++++++++++++ api/swagger.yaml | 2 +- client/image_delta.go | 22 +++++++++++++++++ cmd/dockerd/daemon.go | 2 -- 9 files changed, 47 insertions(+), 76 deletions(-) delete mode 100644 api/server/router/delta/backend.go delete mode 100644 api/server/router/delta/delta.go delete mode 100644 api/server/router/delta/delta_routes.go create mode 100644 client/image_delta.go diff --git a/api/server/router/delta/backend.go b/api/server/router/delta/backend.go deleted file mode 100644 index 0d6e3cedab..0000000000 --- a/api/server/router/delta/backend.go +++ /dev/null @@ -1,11 +0,0 @@ -package delta - -import ( - "io" -) - -// Backend is the methods that need to be implemented to provide -// delta specific functionality. -type Backend interface { - DeltaCreate(deltaSrc, deltaDest string, outStream io.Writer) error -} diff --git a/api/server/router/delta/delta.go b/api/server/router/delta/delta.go deleted file mode 100644 index 93822245a7..0000000000 --- a/api/server/router/delta/delta.go +++ /dev/null @@ -1,30 +0,0 @@ -package delta - -import "github.com/docker/docker/api/server/router" - -// deltaRouter is a router to talk with the deltas controller -type deltaRouter struct { - backend Backend - routes []router.Route -} - -// NewRouter initializes a new delta router -func NewRouter(b Backend) router.Router { - r := &deltaRouter{ - backend: b, - } - r.initRoutes() - return r -} - -// Routes returns the available routes to the deltas controller -func (r *deltaRouter) Routes() []router.Route { - return r.routes -} - -func (r *deltaRouter) initRoutes() { - r.routes = []router.Route{ - // POST - router.NewPostRoute("/deltas/create", r.postDeltasCreate), - } -} diff --git a/api/server/router/delta/delta_routes.go b/api/server/router/delta/delta_routes.go deleted file mode 100644 index 848ed60ca4..0000000000 --- a/api/server/router/delta/delta_routes.go +++ /dev/null @@ -1,32 +0,0 @@ -package delta - -import ( - "net/http" - - "github.com/docker/docker/api/server/httputils" - "github.com/docker/docker/pkg/ioutils" - "github.com/docker/docker/pkg/streamformatter" - "golang.org/x/net/context" -) - -func (d *deltaRouter) postDeltasCreate(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { - if err := httputils.ParseForm(r); err != nil { - return err - } - - deltaSrc := r.Form.Get("src") - deltaDest := r.Form.Get("dest") - - output := ioutils.NewWriteFlusher(w) - defer output.Close() - - w.Header().Set("Content-Type", "application/json") - - if err := d.backend.DeltaCreate(deltaSrc, deltaDest, output); err != nil { - if !output.Flushed() { - return err - } - output.Write(streamformatter.FormatError(err)) - } - return nil -} diff --git a/api/server/router/image/backend.go b/api/server/router/image/backend.go index 9a588a71a9..f3e92a1d6d 100644 --- a/api/server/router/image/backend.go +++ b/api/server/router/image/backend.go @@ -25,6 +25,7 @@ type containerBackend interface { } type imageBackend interface { + DeltaCreate(deltaSrc, deltaDest string, outStream io.Writer) error ImageDelete(imageRef string, force, prune bool) ([]types.ImageDeleteResponseItem, error) ImageHistory(imageName string) ([]*image.HistoryResponseItem, error) Images(imageFilters filters.Args, all bool, withExtraAttrs bool) ([]*types.ImageSummary, error) diff --git a/api/server/router/image/image.go b/api/server/router/image/image.go index 6c233d900c..132e281c74 100644 --- a/api/server/router/image/image.go +++ b/api/server/router/image/image.go @@ -41,6 +41,7 @@ func (r *imageRouter) initRoutes() { router.NewPostRoute("/commit", r.postCommit), router.NewPostRoute("/images/load", r.postImagesLoad), router.NewPostRoute("/images/create", r.postImagesCreate, router.WithCancel), + router.NewPostRoute("/images/delta", r.postImagesDelta), router.NewPostRoute("/images/{name:.*}/push", r.postImagesPush, router.WithCancel), router.NewPostRoute("/images/{name:.*}/tag", r.postImagesTag), router.NewPostRoute("/images/prune", r.postImagesPrune, router.WithCancel), diff --git a/api/server/router/image/image_routes.go b/api/server/router/image/image_routes.go index 9b99a585f3..dd398e47a4 100644 --- a/api/server/router/image/image_routes.go +++ b/api/server/router/image/image_routes.go @@ -161,6 +161,28 @@ func (s *imageRouter) postImagesCreate(ctx context.Context, w http.ResponseWrite return nil } +func (d *imageRouter) postImagesDelta(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { + if err := httputils.ParseForm(r); err != nil { + return err + } + + deltaSrc := r.Form.Get("src") + deltaDest := r.Form.Get("dest") + + output := ioutils.NewWriteFlusher(w) + defer output.Close() + + w.Header().Set("Content-Type", "application/json") + + if err := d.backend.DeltaCreate(deltaSrc, deltaDest, output); err != nil { + if !output.Flushed() { + return err + } + output.Write(streamformatter.FormatError(err)) + } + return nil +} + func (s *imageRouter) postImagesPush(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { metaHeaders := map[string][]string{} for k, v := range r.Header { diff --git a/api/swagger.yaml b/api/swagger.yaml index c29d109d6c..1a58e0f115 100644 --- a/api/swagger.yaml +++ b/api/swagger.yaml @@ -5238,7 +5238,7 @@ paths: schema: $ref: "#/definitions/ErrorResponse" tags: ["Image"] - /deltas/create: + /image/delta: post: summary: "Create a delta" description: "Create a binary delta between two images." diff --git a/client/image_delta.go b/client/image_delta.go new file mode 100644 index 0000000000..1018fd05ab --- /dev/null +++ b/client/image_delta.go @@ -0,0 +1,22 @@ +package client + +import ( + "io" + "net/url" + + "golang.org/x/net/context" +) + +// ImageImport creates a new image based in the source options. +// It returns the JSON content in the response body. +func (cli *Client) ImageDelta(ctx context.Context, src, dest string) (io.ReadCloser, error) { + query := url.Values{} + query.Set("src", src) + query.Set("dest", dest) + + resp, err := cli.postRaw(ctx, "/images/delta", query, nil, nil) + if err != nil { + return nil, err + } + return resp.body, nil +} diff --git a/cmd/dockerd/daemon.go b/cmd/dockerd/daemon.go index 6927534491..893266f1fa 100644 --- a/cmd/dockerd/daemon.go +++ b/cmd/dockerd/daemon.go @@ -19,7 +19,6 @@ import ( "github.com/docker/docker/api/server/router/container" distributionrouter "github.com/docker/docker/api/server/router/distribution" "github.com/docker/docker/api/server/router/image" - deltarouter "github.com/docker/docker/api/server/router/delta" "github.com/docker/docker/api/server/router/network" sessionrouter "github.com/docker/docker/api/server/router/session" systemrouter "github.com/docker/docker/api/server/router/system" @@ -469,7 +468,6 @@ func initRouter(opts routerOptions) { container.NewRouter(opts.daemon, decoder), image.NewRouter(opts.daemon, decoder), systemrouter.NewRouter(opts.daemon, opts.buildCache), - deltarouter.NewRouter(opts.daemon), volume.NewRouter(opts.daemon), build.NewRouter(opts.buildBackend, opts.daemon), sessionrouter.NewRouter(opts.sessionManager), From bd14e2f1121eabd713de5b2399019fe835122cce Mon Sep 17 00:00:00 2001 From: Petros Angelatos Date: Fri, 6 Oct 2017 16:38:05 -0700 Subject: [PATCH 4/5] delta: revert automatic tag generation for delta image Signed-off-by: Petros Angelatos --- daemon/create.go | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/daemon/create.go b/daemon/create.go index 3190e960b0..018a6f9fe9 100644 --- a/daemon/create.go +++ b/daemon/create.go @@ -16,7 +16,6 @@ import ( "github.com/pkg/errors" "github.com/Sirupsen/logrus" - "github.com/docker/distribution/reference" apierrors "github.com/docker/docker/api/errors" "github.com/docker/docker/api/types" containertypes "github.com/docker/docker/api/types/container" @@ -31,7 +30,6 @@ import ( "github.com/docker/docker/pkg/streamformatter" "github.com/docker/docker/pkg/system" "github.com/docker/docker/runconfig" - "github.com/opencontainers/go-digest" "github.com/opencontainers/selinux/go-selinux/label" "github.com/resin-os/librsync-go" ) @@ -521,16 +519,6 @@ func (daemon *Daemon) DeltaCreate(deltaSrc, deltaDest string, outStream io.Write return err } - ref, _ := reference.WithName("delta") - - deltaTag := "delta-" + digest.FromString(srcImg.ID().String() + "-" + dstImg.ImageID()).Hex()[:8] - - ref2, _ := reference.WithTag(ref, deltaTag) - - if err := daemon.TagImageWithReference(id, "linux", ref2); err != nil { - return err - } - outStream.Write(streamformatter.FormatStatus("", id.String())) return nil } From 18d86044b3ab5a36c49b4cd6cde4000d548ac5ae Mon Sep 17 00:00:00 2001 From: Petros Angelatos Date: Wed, 11 Oct 2017 18:21:56 -0700 Subject: [PATCH 5/5] daemon: compute and print summary of delta efficiency Signed-off-by: Petros Angelatos --- daemon/create.go | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/daemon/create.go b/daemon/create.go index 018a6f9fe9..ed2b4bf8d9 100644 --- a/daemon/create.go +++ b/daemon/create.go @@ -30,6 +30,7 @@ import ( "github.com/docker/docker/pkg/streamformatter" "github.com/docker/docker/pkg/system" "github.com/docker/docker/runconfig" + units "github.com/docker/go-units" "github.com/opencontainers/selinux/go-selinux/label" "github.com/resin-os/librsync-go" ) @@ -385,6 +386,9 @@ func (daemon *Daemon) DeltaCreate(deltaSrc, deltaDest string, outStream io.Write progress.Update(progressOutput, stringid.TruncateID(diffID.String()), "Waiting") } + statTotalSize := int64(0) + statDetlaSize := int64(0) + for i, diffID := range dstImg.RootFS.DiffIDs { var ( layerData io.Reader @@ -421,6 +425,8 @@ func (daemon *Daemon) DeltaCreate(deltaSrc, deltaDest string, outStream io.Write return err } + statTotalSize += inputSize + progressReader := progress.NewProgressReader(input, progressOutput, inputSize, stringid.TruncateID(diffID.String()), "Computing delta") defer progressReader.Close() @@ -490,6 +496,11 @@ func (daemon *Daemon) DeltaCreate(deltaSrc, deltaDest string, outStream io.Write if commonLayer { progress.Update(progressOutput, stringid.TruncateID(diffID.String()), "Skipping common layer") } else { + deltaSize, err := newLayer.DiffSize() + if err != nil { + return err + } + statDetlaSize += deltaSize progress.Update(progressOutput, stringid.TruncateID(diffID.String()), "Delta complete") } @@ -519,6 +530,14 @@ func (daemon *Daemon) DeltaCreate(deltaSrc, deltaDest string, outStream io.Write return err } - outStream.Write(streamformatter.FormatStatus("", id.String())) + humanTotal := units.HumanSize(float64(statTotalSize)) + humanDelta := units.HumanSize(float64(statDetlaSize)) + deltaRatio := float64(statTotalSize) / float64(statDetlaSize) + if statTotalSize == 0 { + deltaRatio = 1 + } + + outStream.Write(streamformatter.FormatStatus("", "Normal size: %s, Delta size: %s, %.2fx improvement", humanTotal, humanDelta, deltaRatio)) + outStream.Write(streamformatter.FormatStatus("", "Created delta: %s", id.String())) return nil }