Compute diff from the upper directory of overlayfs-based snapshotter

Signed-off-by: ktock <ktokunaga.mail@gmail.com>

cache/blobs.go

@@ -2,10 +2,22 @@ package cache
 
 import (
 	"context"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"syscall"
 
+	"github.com/containerd/containerd/archive"
+	ctdcompression "github.com/containerd/containerd/archive/compression"
+	"github.com/containerd/containerd/content"
 	"github.com/containerd/containerd/diff"
+	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/leases"
 	"github.com/containerd/containerd/mount"
+	"github.com/containerd/continuity/devices"
+	"github.com/containerd/continuity/fs"
+	"github.com/containerd/continuity/sysx"
 	"github.com/moby/buildkit/session"
 	"github.com/moby/buildkit/util/compression"
 	"github.com/moby/buildkit/util/flightcontrol"
@@ -14,12 +26,17 @@ import (
 	imagespecidentity "github.com/opencontainers/image-spec/identity"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 	"golang.org/x/sync/errgroup"
+	"golang.org/x/sys/unix"
 )
 
 var g flightcontrol.Group
 
-const containerdUncompressed = "containerd.io/uncompressed"
+const (
+	containerdUncompressed     = "containerd.io/uncompressed"
+	containerdUpperDirLabelKey = "containerd.io/snapshot/overlay.upperdir"
+)
 
 var ErrNoBlobs = errors.Errorf("no blobs for snapshot")
 
@@ -73,6 +90,14 @@ func computeBlobChain(ctx context.Context, sr *immutableRef, createIfNeeded bool
 			var descr ocispec.Descriptor
 			var err error
 
+			if descr.Digest == "" && !isTypeWindows(sr) {
+				// Try optimized diff for overlayfs
+				descr, err = sr.computeOverlayBlob(ctx, mediaType, sr.ID(), s)
+				if err != nil {
+					logrus.Errorf("failed to compute blob (%s) from diff of overlay snapshotter: %+v", sr.ID(), err)
+				}
+			}
+
 			if descr.Digest == "" {
 				// reference needs to be committed
 				var lower []mount.Mount
@@ -224,3 +249,170 @@ func isTypeWindows(sr *immutableRef) bool {
 	}
 	return false
 }
+
+var emptyDesc = ocispec.Descriptor{}
+
+func (sr *immutableRef) computeOverlayBlob(ctx context.Context, mediaType string, ref string, s session.Group) (_ ocispec.Descriptor, err error) {
+	sinfo, err := sr.cm.Snapshotter.Stat(ctx, getSnapshotID(sr.md))
+	if err != nil {
+		return emptyDesc, err
+	}
+	upper, ok := sinfo.Labels[containerdUpperDirLabelKey]
+	if !ok {
+		return emptyDesc, fmt.Errorf("upper directory is not registered")
+	}
+
+	var isCompressed bool
+	switch mediaType {
+	case ocispec.MediaTypeImageLayer:
+	case ocispec.MediaTypeImageLayerGzip:
+		isCompressed = true
+	default:
+		return emptyDesc, fmt.Errorf("unsupported diff media type: %v", mediaType)
+	}
+
+	cw, err := sr.cm.ContentStore.Writer(ctx,
+		content.WithRef(ref),
+		content.WithDescriptor(ocispec.Descriptor{
+			MediaType: mediaType, // most contentstore implementations just ignore this
+		}))
+	if err != nil {
+		return emptyDesc, errors.Wrap(err, "failed to open writer")
+	}
+	defer func() {
+		if err != nil {
+			cw.Close()
+		}
+	}()
+
+	var lower []mount.Mount
+	if sr.parent != nil {
+		m, err := sr.parent.Mount(ctx, true, s)
+		if err != nil {
+			return emptyDesc, err
+		}
+		var release func() error
+		lower, release, err = m.Mount()
+		if err != nil {
+			return emptyDesc, err
+		}
+		if release != nil {
+			defer release()
+		}
+	}
+
+	var labels map[string]string
+	if isCompressed {
+		dgstr := digest.SHA256.Digester()
+		compressed, err := ctdcompression.CompressStream(cw, ctdcompression.Gzip)
+		if err != nil {
+			return emptyDesc, errors.Wrap(err, "failed to get compressed stream")
+		}
+		err = writeOverlayUpperdir(ctx, io.MultiWriter(compressed, dgstr.Hash()), upper, lower)
+		compressed.Close()
+		if err != nil {
+			return emptyDesc, errors.Wrap(err, "failed to write compressed diff")
+		}
+		if labels == nil {
+			labels = map[string]string{}
+		}
+		labels[containerdUncompressed] = dgstr.Digest().String()
+	} else {
+		if err = writeOverlayUpperdir(ctx, cw, upper, lower); err != nil {
+			return emptyDesc, errors.Wrap(err, "failed to write diff")
+		}
+	}
+
+	var commitopts []content.Opt
+	if labels != nil {
+		commitopts = append(commitopts, content.WithLabels(labels))
+	}
+	dgst := cw.Digest()
+	if err := cw.Commit(ctx, 0, dgst, commitopts...); err != nil {
+		if !errdefs.IsAlreadyExists(err) {
+			return emptyDesc, errors.Wrap(err, "failed to commit")
+		}
+	}
+	cinfo, err := sr.cm.ContentStore.Info(ctx, dgst)
+	if err != nil {
+		return emptyDesc, errors.Wrap(err, "failed to get info from content store")
+	}
+	if cinfo.Labels == nil {
+		cinfo.Labels = make(map[string]string)
+	}
+	// Set uncompressed label if digest already existed without label
+	if _, ok := cinfo.Labels[containerdUncompressed]; !ok {
+		cinfo.Labels[containerdUncompressed] = labels[containerdUncompressed]
+		if _, err := sr.cm.ContentStore.Update(ctx, cinfo, "labels."+containerdUncompressed); err != nil {
+			return emptyDesc, errors.Wrap(err, "error setting uncompressed label")
+		}
+	}
+
+	return ocispec.Descriptor{
+		MediaType: mediaType,
+		Size:      cinfo.Size,
+		Digest:    cinfo.Digest,
+	}, nil
+}
+
+func writeOverlayUpperdir(ctx context.Context, w io.Writer, upper string, lower []mount.Mount) error {
+	cw := archive.NewChangeWriter(w, upper)
+	changeFn := cw.HandleChange
+	err := mount.WithTempMount(ctx, lower, func(lowerRoot string) error {
+		return fs.DiffDirChanges(ctx, changeFn, lowerRoot, &fs.DiffDirOptions{
+			DiffDir:      upper,
+			DeleteChange: overlayDeleteChange,
+		})
+	})
+	if err != nil {
+		return err
+	}
+	return cw.Close()
+}
+
+func overlayDeleteChange(diffDir string, path string, base string, f os.FileInfo) (deleteFile string, skip bool, err error) {
+	// Check if this is a whiteout
+	if f.Mode()&os.ModeCharDevice != 0 {
+		if _, ok := f.Sys().(*syscall.Stat_t); ok {
+			maj, min, err := devices.DeviceInfo(f)
+			if err != nil {
+				return "", false, err
+			}
+			if maj == 0 && min == 0 {
+				// This file is deleted from base directory
+				if _, err := os.Stat(filepath.Join(base, path)); err != nil {
+					if !os.IsNotExist(err) {
+						return "", false, err
+					}
+					// This file doesn't exist even in the base dir. We don't need whiteout.
+					return "", true, nil
+				}
+				return path, false, nil
+			}
+		}
+	}
+
+	// Check if this is an opaque directory
+	if f.IsDir() {
+		opaque, err := sysx.LGetxattr(filepath.Join(diffDir, path), "trusted.overlay.opaque")
+		if err != nil {
+			if err != unix.ENODATA {
+				return "", false, errors.Wrapf(err, "failed to retrieve trusted.overlay.opaque attr")
+			}
+		} else if len(opaque) == 1 && opaque[0] == 'y' {
+			// Add this directory and an opaque whiteout file.
+			if _, err := os.Stat(filepath.Join(base, path)); err != nil {
+				if !os.IsNotExist(err) {
+					return "", false, err
+				}
+				// This file doesn't exist even in the base dir. We don't need whiteout.
+				return "", true, nil
+			}
+			// NOTE: This is a hack to let HandleChange create an opaque entry (".wh..wh..opq").
+			//       HandleChange creates a whiteout named "<path>/.wh.<filename>" so we pass ".wh..opq" as filename here.
+			return filepath.Join(path, ".wh..opq"), false, nil
+		}
+	}
+
+	return "", false, nil
+}

go.mod

@@ -4,8 +4,8 @@ go 1.13
 
 require (
 	github.com/BurntSushi/toml v0.3.1
-	github.com/Microsoft/go-winio v0.4.17
-	github.com/Microsoft/hcsshim v0.8.16
+	github.com/Microsoft/go-winio v0.5.0
+	github.com/Microsoft/hcsshim v0.8.17
 	github.com/codahale/hdrhistogram v0.0.0-20160425231609-f8ad88b59a58 // indirect
 	github.com/containerd/console v1.0.2
 	github.com/containerd/containerd v1.5.2
@@ -43,9 +43,9 @@ require (
 	github.com/morikuni/aec v1.0.0
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/image-spec v1.0.1
-	github.com/opencontainers/runc v1.0.0-rc93
-	github.com/opencontainers/runtime-spec v1.0.3-0.20200929063507-e6143ca7d51d
-	github.com/opencontainers/selinux v1.8.0
+	github.com/opencontainers/runc v1.0.0-rc95
+	github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417
+	github.com/opencontainers/selinux v1.8.2
 	github.com/opentracing-contrib/go-stdlib v1.0.0
 	github.com/opentracing/opentracing-go v1.2.0
 	github.com/pkg/errors v0.9.1
@@ -62,7 +62,7 @@ require (
 	golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2
 	golang.org/x/net v0.0.0-20210226172049-e18ecbb05110
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
-	golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57
+	golang.org/x/sys v0.0.0-20210426230700-d19ff857e887
 	golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e
 	// genproto: the actual version is replaced in replace()
 	google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a
@@ -71,6 +71,9 @@ require (
 )
 
 replace (
+	// Patched version of containerd and continuity for overlayfs-optimized differ
+	github.com/containerd/containerd => github.com/ktock/containerd v1.2.1-0.20210615124510-50ed319ed864
+	github.com/containerd/continuity => github.com/ktock/continuity v0.1.1-0.20210616010202-ce76fccd7658
 	github.com/docker/docker => github.com/docker/docker v20.10.3-0.20210609100121-ef4d47340142+incompatible
 	// protobuf: corresponds to containerd
 	github.com/golang/protobuf => github.com/golang/protobuf v1.3.5