From f10537038b1dcebd18b104e8ac5da415d1cb43dc Mon Sep 17 00:00:00 2001 From: Kohei Tokunaga Date: Mon, 30 Aug 2021 17:26:40 +0900 Subject: [PATCH] Compute diff from the upper dir of overlayfs-based snapshotter Signed-off-by: Kohei Tokunaga --- cache/blobs.go | 40 ++- cache/blobs_linux.go | 417 ++++++++++++++++++++++++ cache/blobs_linux_test.go | 458 +++++++++++++++++++++++++++ cache/blobs_nolinux.go | 15 + util/testutil/integration/sandbox.go | 2 +- 5 files changed, 924 insertions(+), 8 deletions(-) create mode 100644 cache/blobs_linux.go create mode 100644 cache/blobs_linux_test.go create mode 100644 cache/blobs_nolinux.go diff --git a/cache/blobs.go b/cache/blobs.go index 5c3c123d38fe4..f62bc5065ca5e 100644 --- a/cache/blobs.go +++ b/cache/blobs.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "io" + "os" "github.com/containerd/containerd/content" "github.com/containerd/containerd/diff" @@ -17,6 +18,7 @@ import ( imagespecidentity "github.com/opencontainers/image-spec/identity" ocispecs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "golang.org/x/sync/errgroup" ) @@ -106,13 +108,37 @@ func computeBlobChain(ctx context.Context, sr *immutableRef, createIfNeeded bool if release != nil { defer release() } - desc, err := sr.cm.Differ.Compare(ctx, lower, upper, - diff.WithMediaType(mediaType), - diff.WithReference(sr.ID()), - diff.WithCompressor(compressorFunc), - ) - if err != nil { - return nil, err + var desc ocispecs.Descriptor + if len(lower) > 0 && !isTypeWindows(sr) { + // Try optimized diff for overlayfs + computed, ok, err := sr.tryComputeOverlayBlob(ctx, lower, upper, mediaType, sr.ID(), compressorFunc) + if err != nil { + logrus.Warnf("failed to compute blob (%s) from diff of overlay snapshotter: %+v", sr.ID(), err) + } + if ok { + desc = computed + } + if desc.Digest == "" { + switch sr.cm.ManagerOpt.Snapshotter.Name() { + case "overlayfs", "fuse-overlayfs", "stargz": + if os.Getenv("BUILDKIT_DEBUG_FORCE_OVERLAY_DIFF") == "1" { + // Fallback is disabled. used for testing/debugging. + return nil, errors.Errorf("failed to compute overlay blob (ok=%v): %v", ok, err) + } + // Print warn log if we use the snapshotter expected to succeed overlayfs-based differ + logrus.Warnf("failed to compute overlay blob (ok=%v): %v", ok, err) + } + } + } + if desc.Digest == "" { + desc, err = sr.cm.Differ.Compare(ctx, lower, upper, + diff.WithMediaType(mediaType), + diff.WithReference(sr.ID()), + diff.WithCompressor(compressorFunc), + ) + if err != nil { + return nil, err + } } if desc.Annotations == nil { diff --git a/cache/blobs_linux.go b/cache/blobs_linux.go new file mode 100644 index 0000000000000..10f207c5bb855 --- /dev/null +++ b/cache/blobs_linux.go @@ -0,0 +1,417 @@ +// +build linux + +package cache + +import ( + "bytes" + "context" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "strings" + "syscall" + + "github.com/containerd/containerd/archive" + ctdcompression "github.com/containerd/containerd/archive/compression" + "github.com/containerd/containerd/content" + "github.com/containerd/containerd/errdefs" + "github.com/containerd/containerd/mount" + "github.com/containerd/continuity/devices" + "github.com/containerd/continuity/fs" + "github.com/containerd/continuity/sysx" + digest "github.com/opencontainers/go-digest" + ocispecs "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" + "golang.org/x/sys/unix" +) + +var emptyDesc = ocispecs.Descriptor{} + +// computeOverlayBlob provides overlayfs-specialized method to compute +// diff between lower and upper snapshot. If the passed mounts cannot +// be computed (e.g. because the mounts aren't overlayfs), it returns +// an error. +func (sr *immutableRef) tryComputeOverlayBlob(ctx context.Context, lower, upper []mount.Mount, mediaType string, ref string, compressorFunc compressor) (_ ocispecs.Descriptor, ok bool, err error) { + + // Get upperdir location if mounts are overlayfs that can be processed by this differ. + upperdir, err := getOverlayUpperdir(lower, upper) + if err != nil { + // This is not an overlayfs snapshot. This is not an error so don't return error here + // and let the caller fallback to another differ. + return emptyDesc, false, nil + } + + if compressorFunc == nil { + switch mediaType { + case ocispecs.MediaTypeImageLayer: + case ocispecs.MediaTypeImageLayerGzip: + compressorFunc = func(dest io.Writer, requiredMediaType string) (io.WriteCloser, error) { + return ctdcompression.CompressStream(dest, ctdcompression.Gzip) + } + default: + return emptyDesc, false, errors.Errorf("unsupported diff media type: %v", mediaType) + } + } + + cw, err := sr.cm.ContentStore.Writer(ctx, + content.WithRef(ref), + content.WithDescriptor(ocispecs.Descriptor{ + MediaType: mediaType, // most contentstore implementations just ignore this + })) + if err != nil { + return emptyDesc, false, errors.Wrap(err, "failed to open writer") + } + defer func() { + if err != nil { + cw.Close() + } + }() + + var labels map[string]string + if compressorFunc != nil { + dgstr := digest.SHA256.Digester() + compressed, err := compressorFunc(cw, mediaType) + if err != nil { + return emptyDesc, false, errors.Wrap(err, "failed to get compressed stream") + } + err = writeOverlayUpperdir(ctx, io.MultiWriter(compressed, dgstr.Hash()), upperdir, lower) + compressed.Close() + if err != nil { + return emptyDesc, false, errors.Wrap(err, "failed to write compressed diff") + } + if labels == nil { + labels = map[string]string{} + } + labels[containerdUncompressed] = dgstr.Digest().String() + } else { + if err = writeOverlayUpperdir(ctx, cw, upperdir, lower); err != nil { + return emptyDesc, false, errors.Wrap(err, "failed to write diff") + } + } + + var commitopts []content.Opt + if labels != nil { + commitopts = append(commitopts, content.WithLabels(labels)) + } + dgst := cw.Digest() + if err := cw.Commit(ctx, 0, dgst, commitopts...); err != nil { + if !errdefs.IsAlreadyExists(err) { + return emptyDesc, false, errors.Wrap(err, "failed to commit") + } + } + cinfo, err := sr.cm.ContentStore.Info(ctx, dgst) + if err != nil { + return emptyDesc, false, errors.Wrap(err, "failed to get info from content store") + } + if cinfo.Labels == nil { + cinfo.Labels = make(map[string]string) + } + // Set uncompressed label if digest already existed without label + if _, ok := cinfo.Labels[containerdUncompressed]; !ok { + cinfo.Labels[containerdUncompressed] = labels[containerdUncompressed] + if _, err := sr.cm.ContentStore.Update(ctx, cinfo, "labels."+containerdUncompressed); err != nil { + return emptyDesc, false, errors.Wrap(err, "error setting uncompressed label") + } + } + + return ocispecs.Descriptor{ + MediaType: mediaType, + Size: cinfo.Size, + Digest: cinfo.Digest, + }, true, nil +} + +// getOverlayUpperdir parses the passed mounts and identifies the directory +// that contains diff between upper and lower. +func getOverlayUpperdir(lower, upper []mount.Mount) (string, error) { + var upperdir string + if len(lower) == 1 && len(upper) == 1 { + // We only support single mount configuration as of now. + upperM, lowerM := upper[0], lower[0] + + // Get layer directories of lower snapshot + var lowerlayers []string + switch lowerM.Type { + case "bind": + // lower snapshot is a bind mount of one layer + lowerlayers = []string{lowerM.Source} + case "overlay": + // lower snapshot is an overlay mount of multiple layers + var err error + lowerlayers, err = getOverlayLayers(lowerM) + if err != nil { + return "", err + } + default: + return "", errors.Errorf("cannot get layer information from mount option (type = %q)", lowerM.Type) + } + + // Get layer directories of upper snapshot + if upperM.Type != "overlay" { + return "", errors.Errorf("upper snapshot isn't overlay mounted (type = %q)", upperM.Type) + } + upperlayers, err := getOverlayLayers(upperM) + if err != nil { + return "", err + } + + // Check if the diff directory can be determined + if len(upperlayers) != len(lowerlayers)+1 { + return "", errors.Errorf("cannot determine diff of more than one upper directories") + } + for i := 0; i < len(lowerlayers); i++ { + if upperlayers[i] != lowerlayers[i] { + return "", errors.Errorf("layer %d must be common between upper and lower snapshots", i) + } + } + upperdir = upperlayers[len(upperlayers)-1] // get the topmost layer that indicates diff + } else { + return "", errors.Errorf("multiple mount configurations are not supported") + } + if upperdir == "" { + return "", errors.Errorf("cannot determine upperdir from mount option") + } + return upperdir, nil +} + +// getOverlayLayers returns all layer directories of an overlayfs mount. +func getOverlayLayers(m mount.Mount) ([]string, error) { + var u string + var uFound bool + var l []string // l[0] = bottommost + for _, o := range m.Options { + if strings.HasPrefix(o, "upperdir=") { + u, uFound = strings.TrimPrefix(o, "upperdir="), true + } else if strings.HasPrefix(o, "lowerdir=") { + l = strings.Split(strings.TrimPrefix(o, "lowerdir="), ":") + for i, j := 0, len(l)-1; i < j; i, j = i+1, j-1 { + l[i], l[j] = l[j], l[i] // make l[0] = bottommost + } + } else if strings.HasPrefix(o, "workdir=") || o == "index=off" || o == "userxattr" { + // these options are possible to specfied by the snapshotter but not indicate dir locations. + continue + } else { + // encountering an unknown option. return error and fallback to walking differ + // to avoid unexpected diff. + return nil, errors.Errorf("unknown option %q specified by snapshotter", o) + } + } + if uFound { + return append(l, u), nil + } + return l, nil +} + +// writeOverlayUpperdir writes a layer tar archive into the specified writer, based on +// the diff information stored in the upperdir. +func writeOverlayUpperdir(ctx context.Context, w io.Writer, upperdir string, lower []mount.Mount) error { + emptyLower, err := ioutil.TempDir("", "buildkit") // empty directory used for the lower of diff view + if err != nil { + return errors.Wrapf(err, "failed to create temp dir") + } + defer os.Remove(emptyLower) + upperView := []mount.Mount{ + { + Type: "overlay", + Source: "overlay", + Options: []string{fmt.Sprintf("lowerdir=%s", strings.Join([]string{upperdir, emptyLower}, ":"))}, + }, + } + return mount.WithTempMount(ctx, lower, func(lowerRoot string) error { + return mount.WithTempMount(ctx, upperView, func(upperViewRoot string) error { + cw := archive.NewChangeWriter(w, upperViewRoot) + if err := overlayChanges(ctx, cw.HandleChange, upperdir, upperViewRoot, lowerRoot); err != nil { + if err2 := cw.Close(); err2 != nil { + return errors.Wrapf(err, "failed torecord upperdir changes (close error: %v)", err2) + } + return errors.Wrapf(err, "failed torecord upperdir changes") + } + return cw.Close() + }) + }) +} + +// overlayChanges is continuty's `fs.Change`-like method but leverages overlayfs's +// "upperdir" for computing the diff. "upperdirView" is overlayfs mounted view of +// the upperdir that doesn't contain whiteouts. This is used for computing +// changes under opaque directories. +func overlayChanges(ctx context.Context, changeFn fs.ChangeFunc, upperdir, upperdirView, base string) error { + return filepath.Walk(upperdir, func(path string, f os.FileInfo, err error) error { + if err != nil { + return err + } + + // Rebase path + path, err = filepath.Rel(upperdir, path) + if err != nil { + return err + } + path = filepath.Join(string(os.PathSeparator), path) + + // Skip root + if path == string(os.PathSeparator) { + return nil + } + + // Check if this is a deleted entry + isDelete, skip, err := checkDelete(upperdir, path, base, f) + if err != nil { + return err + } else if skip { + return nil + } + + var kind fs.ChangeKind + var skipRecord bool + if isDelete { + // This is a deleted entry. + kind = fs.ChangeKindDelete + f = nil + } else if baseF, err := os.Lstat(filepath.Join(base, path)); err == nil { + // File exists in the base layer. Thus this is modified. + kind = fs.ChangeKindModify + // Avoid including directory that hasn't been modified. If /foo/bar/baz is modified, + // then /foo will apper here even if it's not been modified because it's the parent of bar. + if same, err := sameDir(baseF, f, filepath.Join(base, path), filepath.Join(upperdirView, path)); same { + skipRecord = true // Both are the same, don't record the change + } else if err != nil { + return err + } + } else if os.IsNotExist(err) { + // File doesn't exist in the base layer. Thus this is added. + kind = fs.ChangeKindAdd + } else if err != nil { + return err + } + + if !skipRecord { + if err := changeFn(kind, path, f, nil); err != nil { + return err + } + } + + if f != nil { + if isOpaque, err := checkOpaque(upperdir, path, base, f); err != nil { + return err + } else if isOpaque { + // This is an opaque directory. Start a new walking differ to get adds/deletes of + // this directory. We use "upperdirView" directory which doesn't contain whiteouts. + if err := fs.Changes(ctx, filepath.Join(base, path), filepath.Join(upperdirView, path), + func(k fs.ChangeKind, p string, f os.FileInfo, err error) error { + return changeFn(k, filepath.Join(path, p), f, err) // rebase path to be based on the opaque dir + }, + ); err != nil { + return err + } + return filepath.SkipDir // We completed this directory. Do not walk files under this directory anymore. + } + } + return nil + }) +} + +// checkDelete checks if the specified file is a whiteout +func checkDelete(upperdir string, path string, base string, f os.FileInfo) (delete, skip bool, _ error) { + if f.Mode()&os.ModeCharDevice != 0 { + if _, ok := f.Sys().(*syscall.Stat_t); ok { + maj, min, err := devices.DeviceInfo(f) + if err != nil { + return false, false, errors.Wrapf(err, "failed to get device info") + } + if maj == 0 && min == 0 { + // This file is a whiteout (char 0/0) that indicates this is deleted from the base + if _, err := os.Lstat(filepath.Join(base, path)); err != nil { + if !os.IsNotExist(err) { + return false, false, errors.Wrapf(err, "failed to lstat") + } + // This file doesn't exist even in the base dir. + // We don't need whiteout. Just skip this file. + return false, true, nil + } + return true, false, nil + } + } + } + return false, false, nil +} + +// checkDelete checks if the specified file is an opaque directory +func checkOpaque(upperdir string, path string, base string, f os.FileInfo) (isOpaque bool, _ error) { + if f.IsDir() { + for _, oKey := range []string{"trusted.overlay.opaque", "user.overlay.opaque"} { + opaque, err := sysx.LGetxattr(filepath.Join(upperdir, path), oKey) + if err != nil && err != unix.ENODATA { + return false, errors.Wrapf(err, "failed to retrieve %s attr", oKey) + } else if len(opaque) == 1 && opaque[0] == 'y' { + // This is an opaque whiteout directory. + if _, err := os.Lstat(filepath.Join(base, path)); err != nil { + if !os.IsNotExist(err) { + return false, errors.Wrapf(err, "failed to lstat") + } + // This file doesn't exist even in the base dir. We don't need treat this as an opaque. + return false, nil + } + return true, nil + } + } + } + return false, nil +} + +// sameDir performs continity-compatible comparison of directories. +// https://github.com/containerd/continuity/blob/v0.1.0/fs/path.go#L91-L133 +// This doesn't compare files because it requires to compare their contents. +// This is what we want to avoid by this overlayfs-specialized differ. +func sameDir(f1, f2 os.FileInfo, f1fullPath, f2fullPath string) (bool, error) { + if !f1.IsDir() || !f2.IsDir() { + return false, nil + } + + if os.SameFile(f1, f2) { + return true, nil + } + + equalStat, err := compareSysStat(f1.Sys(), f2.Sys()) + if err != nil || !equalStat { + return equalStat, err + } + + if eq, err := compareCapabilities(f1fullPath, f2fullPath); err != nil || !eq { + return eq, err + } + + return true, nil +} + +// Ported from continuity project +// https://github.com/containerd/continuity/blob/v0.1.0/fs/diff_unix.go#L43-L54 +// Copyright The containerd Authors. +func compareSysStat(s1, s2 interface{}) (bool, error) { + ls1, ok := s1.(*syscall.Stat_t) + if !ok { + return false, nil + } + ls2, ok := s2.(*syscall.Stat_t) + if !ok { + return false, nil + } + + return ls1.Mode == ls2.Mode && ls1.Uid == ls2.Uid && ls1.Gid == ls2.Gid && ls1.Rdev == ls2.Rdev, nil +} + +// Ported from continuity project +// https://github.com/containerd/continuity/blob/v0.1.0/fs/diff_unix.go#L56-L66 +// Copyright The containerd Authors. +func compareCapabilities(p1, p2 string) (bool, error) { + c1, err := sysx.LGetxattr(p1, "security.capability") + if err != nil && err != sysx.ENODATA { + return false, errors.Wrapf(err, "failed to get xattr for %s", p1) + } + c2, err := sysx.LGetxattr(p2, "security.capability") + if err != nil && err != sysx.ENODATA { + return false, errors.Wrapf(err, "failed to get xattr for %s", p2) + } + return bytes.Equal(c1, c2), nil +} diff --git a/cache/blobs_linux_test.go b/cache/blobs_linux_test.go new file mode 100644 index 0000000000000..96d23ed92a7f7 --- /dev/null +++ b/cache/blobs_linux_test.go @@ -0,0 +1,458 @@ +// +build linux + +package cache + +import ( + "context" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/containerd/containerd/mount" + "github.com/containerd/continuity/fs" + "github.com/containerd/continuity/fs/fstest" + "github.com/pkg/errors" +) + +// This test file contains tests that are required in continuity project. +// (https://github.com/containerd/continuity/blob/v0.1.0/fs/diff_test.go) +// Most of them are ported from that project and patched to test our +// overlayfs-optimized differ. + +// TestSimpleDiff is a test ported from +// https://github.com/containerd/continuity/blob/v0.1.0/fs/diff_test.go#L46-L73 +// Copyright The containerd Authors. +func TestSimpleDiff(t *testing.T) { + l1 := fstest.Apply( + fstest.CreateDir("/etc", 0755), + fstest.CreateFile("/etc/hosts", []byte("mydomain 10.0.0.1"), 0644), + fstest.CreateFile("/etc/profile", []byte("PATH=/usr/bin"), 0644), + fstest.CreateFile("/etc/unchanged", []byte("PATH=/usr/bin"), 0644), + fstest.CreateFile("/etc/unexpected", []byte("#!/bin/sh"), 0644), + ) + l2 := fstest.Apply( + fstest.CreateFile("/etc/hosts", []byte("mydomain 10.0.0.120"), 0644), + fstest.CreateFile("/etc/profile", []byte("PATH=/usr/bin"), 0666), + fstest.CreateDir("/root", 0700), + fstest.CreateFile("/root/.bashrc", []byte("PATH=/usr/sbin:/usr/bin"), 0644), + fstest.Remove("/etc/unexpected"), + ) + diff := []TestChange{ + Modify("/etc/hosts"), + Modify("/etc/profile"), + Delete("/etc/unexpected"), + Add("/root"), + Add("/root/.bashrc"), + } + + if err := testDiffWithBase(l1, l2, diff); err != nil { + t.Fatalf("Failed diff with base: %+v", err) + } +} + +// TestEmptyFileDiff is a test ported from +// https://github.com/containerd/continuity/blob/v0.1.0/fs/diff_test.go#L75-L89 +// Copyright The containerd Authors. +func TestEmptyFileDiff(t *testing.T) { + tt := time.Now().Truncate(time.Second) + l1 := fstest.Apply( + fstest.CreateDir("/etc", 0755), + fstest.CreateFile("/etc/empty", []byte(""), 0644), + fstest.Chtimes("/etc/empty", tt, tt), + ) + l2 := fstest.Apply() + diff := []TestChange{} + + if err := testDiffWithBase(l1, l2, diff); err != nil { + t.Fatalf("Failed diff with base: %+v", err) + } +} + +// TestNestedDeletion is a test ported from +// https://github.com/containerd/continuity/blob/v0.1.0/fs/diff_test.go#L91-L111 +// Copyright The containerd Authors. +func TestNestedDeletion(t *testing.T) { + l1 := fstest.Apply( + fstest.CreateDir("/d0", 0755), + fstest.CreateDir("/d1", 0755), + fstest.CreateDir("/d1/d2", 0755), + fstest.CreateFile("/d1/d2/f1", []byte("mydomain 10.0.0.1"), 0644), + ) + l2 := fstest.Apply( + fstest.RemoveAll("/d0"), + fstest.RemoveAll("/d1"), + ) + diff := []TestChange{ + Delete("/d0"), + Delete("/d1"), + } + + if err := testDiffWithBase(l1, l2, diff); err != nil { + t.Fatalf("Failed diff with base: %+v", err) + } +} + +// TestDirectoryReplace is a test ported from +// https://github.com/containerd/continuity/blob/v0.1.0/fs/diff_test.go#L113-L134 +// Copyright The containerd Authors. +func TestDirectoryReplace(t *testing.T) { + l1 := fstest.Apply( + fstest.CreateDir("/dir1", 0755), + fstest.CreateFile("/dir1/f1", []byte("#####"), 0644), + fstest.CreateDir("/dir1/f2", 0755), + fstest.CreateFile("/dir1/f2/f3", []byte("#!/bin/sh"), 0644), + ) + l2 := fstest.Apply( + fstest.CreateFile("/dir1/f11", []byte("#New file here"), 0644), + fstest.RemoveAll("/dir1/f2"), + fstest.CreateFile("/dir1/f2", []byte("Now file"), 0666), + ) + diff := []TestChange{ + Add("/dir1/f11"), + Modify("/dir1/f2"), + } + + if err := testDiffWithBase(l1, l2, diff); err != nil { + t.Fatalf("Failed diff with base: %+v", err) + } +} + +// TestRemoveDirectoryTree is a test ported from +// https://github.com/containerd/continuity/blob/v0.1.0/fs/diff_test.go#L136-L152 +// Copyright The containerd Authors. +func TestRemoveDirectoryTree(t *testing.T) { + l1 := fstest.Apply( + fstest.CreateDir("/dir1/dir2/dir3", 0755), + fstest.CreateFile("/dir1/f1", []byte("f1"), 0644), + fstest.CreateFile("/dir1/dir2/f2", []byte("f2"), 0644), + ) + l2 := fstest.Apply( + fstest.RemoveAll("/dir1"), + ) + diff := []TestChange{ + Delete("/dir1"), + } + + if err := testDiffWithBase(l1, l2, diff); err != nil { + t.Fatalf("Failed diff with base: %+v", err) + } +} + +// TestRemoveDirectoryTreeWithDash is a test ported from +// https://github.com/containerd/continuity/blob/v0.1.0/fs/diff_test.go#L154-L172 +// Copyright The containerd Authors. +func TestRemoveDirectoryTreeWithDash(t *testing.T) { + l1 := fstest.Apply( + fstest.CreateDir("/dir1/dir2/dir3", 0755), + fstest.CreateFile("/dir1/f1", []byte("f1"), 0644), + fstest.CreateFile("/dir1/dir2/f2", []byte("f2"), 0644), + fstest.CreateDir("/dir1-before", 0755), + fstest.CreateFile("/dir1-before/f2", []byte("f2"), 0644), + ) + l2 := fstest.Apply( + fstest.RemoveAll("/dir1"), + ) + diff := []TestChange{ + Delete("/dir1"), + } + + if err := testDiffWithBase(l1, l2, diff); err != nil { + t.Fatalf("Failed diff with base: %+v", err) + } +} + +// TestFileReplace is a test ported from +// https://github.com/containerd/continuity/blob/v0.1.0/fs/diff_test.go#L174-L192 +// Copyright The containerd Authors. +func TestFileReplace(t *testing.T) { + l1 := fstest.Apply( + fstest.CreateFile("/dir1", []byte("a file, not a directory"), 0644), + ) + l2 := fstest.Apply( + fstest.Remove("/dir1"), + fstest.CreateDir("/dir1/dir2", 0755), + fstest.CreateFile("/dir1/dir2/f1", []byte("also a file"), 0644), + ) + diff := []TestChange{ + Modify("/dir1"), + Add("/dir1/dir2"), + Add("/dir1/dir2/f1"), + } + + if err := testDiffWithBase(l1, l2, diff); err != nil { + t.Fatalf("Failed diff with base: %+v", err) + } +} + +// TestParentDirectoryPermission is a test ported from +// https://github.com/containerd/continuity/blob/v0.1.0/fs/diff_test.go#L194-L219 +// Copyright The containerd Authors. +func TestParentDirectoryPermission(t *testing.T) { + l1 := fstest.Apply( + fstest.CreateDir("/dir1", 0700), + fstest.CreateDir("/dir2", 0751), + fstest.CreateDir("/dir3", 0777), + ) + l2 := fstest.Apply( + fstest.CreateDir("/dir1/d", 0700), + fstest.CreateFile("/dir1/d/f", []byte("irrelevant"), 0644), + fstest.CreateFile("/dir1/f", []byte("irrelevant"), 0644), + fstest.CreateFile("/dir2/f", []byte("irrelevant"), 0644), + fstest.CreateFile("/dir3/f", []byte("irrelevant"), 0644), + ) + diff := []TestChange{ + Add("/dir1/d"), + Add("/dir1/d/f"), + Add("/dir1/f"), + Add("/dir2/f"), + Add("/dir3/f"), + } + + if err := testDiffWithBase(l1, l2, diff); err != nil { + t.Fatalf("Failed diff with base: %+v", err) + } +} + +// TestUpdateWithSameTime is a test ported from +// https://github.com/containerd/continuity/blob/v0.1.0/fs/diff_test.go#L221-L269 +// Copyright The containerd Authors. +// +// NOTE: This test is patched for our differ. See the following NOTE for details. +func TestUpdateWithSameTime(t *testing.T) { + tt := time.Now().Truncate(time.Second) + t1 := tt.Add(5 * time.Nanosecond) + t2 := tt.Add(6 * time.Nanosecond) + l1 := fstest.Apply( + fstest.CreateFile("/file-modified-time", []byte("1"), 0644), + fstest.Chtimes("/file-modified-time", t1, t1), + fstest.CreateFile("/file-no-change", []byte("1"), 0644), + fstest.Chtimes("/file-no-change", t1, t1), + fstest.CreateFile("/file-same-time", []byte("1"), 0644), + fstest.Chtimes("/file-same-time", t1, t1), + fstest.CreateFile("/file-truncated-time-1", []byte("1"), 0644), + fstest.Chtimes("/file-truncated-time-1", tt, tt), + fstest.CreateFile("/file-truncated-time-2", []byte("1"), 0644), + fstest.Chtimes("/file-truncated-time-2", tt, tt), + fstest.CreateFile("/file-truncated-time-3", []byte("1"), 0644), + fstest.Chtimes("/file-truncated-time-3", t1, t1), + ) + l2 := fstest.Apply( + fstest.CreateFile("/file-modified-time", []byte("2"), 0644), + fstest.Chtimes("/file-modified-time", t2, t2), + fstest.CreateFile("/file-no-change", []byte("1"), 0644), + fstest.Chtimes("/file-no-change", t1, t1), + fstest.CreateFile("/file-same-time", []byte("2"), 0644), + fstest.Chtimes("/file-same-time", t1, t1), + fstest.CreateFile("/file-truncated-time-1", []byte("1"), 0644), + fstest.Chtimes("/file-truncated-time-1", t1, t1), + fstest.CreateFile("/file-truncated-time-2", []byte("2"), 0644), + fstest.Chtimes("/file-truncated-time-2", tt, tt), + fstest.CreateFile("/file-truncated-time-3", []byte("1"), 0644), + fstest.Chtimes("/file-truncated-time-3", tt, tt), + ) + diff := []TestChange{ + Modify("/file-modified-time"), + + // NOTE: Even if the file is identical, overlayfs copies it to + // the upper layer when the modification occurred. continuity's differ avoids counting + // this as "modify" by comparing the time and the file contents between upper and lower + // but here we want to avoid comparing bits which makes the differ slower. + // TODO: we need a way to effectively determine two files are identical + // without copmaring bits. + Modify("/file-no-change"), + Modify("/file-same-time"), + + // Include changes with truncated timestamps. Comparing newly + // extracted tars which have truncated timestamps will be + // expected to produce changes. The expectation is that diff + // archives are generated once and kept, newly generated diffs + // will not consider cases where only one side is truncated. + Modify("/file-truncated-time-1"), + Modify("/file-truncated-time-2"), + Modify("/file-truncated-time-3"), + } + + if err := testDiffWithBase(l1, l2, diff); err != nil { + t.Fatalf("Failed diff with base: %+v", err) + } +} + +// TestLchtimes is a test ported from +// https://github.com/containerd/continuity/blob/v0.1.0/fs/diff_test.go#L271-L291 +// Copyright The containerd Authors. +// buildkit#172 +func TestLchtimes(t *testing.T) { + mtimes := []time.Time{ + time.Unix(0, 0), // nsec is 0 + time.Unix(0, 42), // nsec > 0 + } + for _, mtime := range mtimes { + atime := time.Unix(424242, 42) + l1 := fstest.Apply( + fstest.CreateFile("/foo", []byte("foo"), 0644), + fstest.Symlink("/foo", "/lnk0"), + fstest.Lchtimes("/lnk0", atime, mtime), + ) + l2 := fstest.Apply() // empty + diff := []TestChange{} + if err := testDiffWithBase(l1, l2, diff); err != nil { + t.Fatalf("Failed diff with base: %+v", err) + } + } +} + +func testDiffWithBase(base, diff fstest.Applier, expected []TestChange) error { + t1, err := ioutil.TempDir("", "diff-with-base-lower-") + if err != nil { + return errors.Wrap(err, "failed to create temp dir") + } + defer os.RemoveAll(t1) + + if err := base.Apply(t1); err != nil { + return errors.Wrap(err, "failed to apply base filesystem") + } + + tupper, err := ioutil.TempDir("", "diff-with-base-upperdir-") + if err != nil { + return errors.Wrap(err, "failed to create temp dir") + } + defer os.RemoveAll(tupper) + + workdir, err := ioutil.TempDir("", "diff-with-base-workdir-") + if err != nil { + return errors.Wrap(err, "failed to create temp dir") + } + defer os.RemoveAll(workdir) + + return mount.WithTempMount(context.Background(), []mount.Mount{ + { + Type: "overlay", + Source: "overlay", + Options: []string{fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", t1, tupper, workdir)}, + }, + }, func(overlayRoot string) error { + if err := diff.Apply(overlayRoot); err != nil { + return errors.Wrapf(err, "failed to apply diff to overlayRoot") + } + if err := collectAndCheckChanges(t1, tupper, expected); err != nil { + return errors.Wrap(err, "failed to collect changes") + } + return nil + }) +} + +func checkChanges(root string, changes, expected []TestChange) error { + if len(changes) != len(expected) { + return errors.Errorf("Unexpected number of changes:\n%s", diffString(changes, expected)) + } + for i := range changes { + if changes[i].Path != expected[i].Path || changes[i].Kind != expected[i].Kind { + return errors.Errorf("Unexpected change at %d:\n%s", i, diffString(changes, expected)) + } + if changes[i].Kind != fs.ChangeKindDelete { + filename := filepath.Join(root, changes[i].Path) + efi, err := os.Stat(filename) + if err != nil { + return errors.Wrapf(err, "failed to stat %q", filename) + } + afi := changes[i].FileInfo + if afi.Size() != efi.Size() { + return errors.Errorf("Unexpected change size %d, %q has size %d", afi.Size(), filename, efi.Size()) + } + if afi.Mode() != efi.Mode() { + return errors.Errorf("Unexpected change mode %s, %q has mode %s", afi.Mode(), filename, efi.Mode()) + } + if afi.ModTime() != efi.ModTime() { + return errors.Errorf("Unexpected change modtime %s, %q has modtime %s", afi.ModTime(), filename, efi.ModTime()) + } + if expected := filepath.Join(root, changes[i].Path); changes[i].Source != expected { + return errors.Errorf("Unexpected source path %s, expected %s", changes[i].Source, expected) + } + } + } + + return nil +} + +type TestChange struct { + Kind fs.ChangeKind + Path string + FileInfo os.FileInfo + Source string +} + +func collectAndCheckChanges(base, upperdir string, expected []TestChange) error { + ctx := context.Background() + changes := []TestChange{} + + emptyLower, err := ioutil.TempDir("", "buildkit-test-emptylower") // empty directory used for the lower of diff view + if err != nil { + return errors.Wrapf(err, "failed to create temp dir") + } + defer os.Remove(emptyLower) + upperView := []mount.Mount{ + { + Type: "overlay", + Source: "overlay", + Options: []string{fmt.Sprintf("lowerdir=%s", strings.Join([]string{upperdir, emptyLower}, ":"))}, + }, + } + return mount.WithTempMount(ctx, upperView, func(upperViewRoot string) error { + if err := overlayChanges(ctx, func(k fs.ChangeKind, p string, f os.FileInfo, err error) error { + if err != nil { + return err + } + changes = append(changes, TestChange{ + Kind: k, + Path: p, + FileInfo: f, + Source: filepath.Join(upperViewRoot, p), + }) + return nil + }, upperdir, upperViewRoot, base); err != nil { + return err + } + if err := checkChanges(upperViewRoot, changes, expected); err != nil { + return errors.Wrapf(err, "change check falied") + } + return nil + }) +} + +func diffString(c1, c2 []TestChange) string { + return fmt.Sprintf("got(%d):\n%s\nexpected(%d):\n%s", len(c1), changesString(c1), len(c2), changesString(c2)) + +} + +func changesString(c []TestChange) string { + strs := make([]string, len(c)) + for i := range c { + strs[i] = fmt.Sprintf("\t%s\t%s", c[i].Kind, c[i].Path) + } + return strings.Join(strs, "\n") +} + +func Add(p string) TestChange { + return TestChange{ + Kind: fs.ChangeKindAdd, + Path: filepath.FromSlash(p), + } +} + +func Delete(p string) TestChange { + return TestChange{ + Kind: fs.ChangeKindDelete, + Path: filepath.FromSlash(p), + } +} + +func Modify(p string) TestChange { + return TestChange{ + Kind: fs.ChangeKindModify, + Path: filepath.FromSlash(p), + } +} diff --git a/cache/blobs_nolinux.go b/cache/blobs_nolinux.go new file mode 100644 index 0000000000000..e8f0bf0aa77ab --- /dev/null +++ b/cache/blobs_nolinux.go @@ -0,0 +1,15 @@ +// +build !linux + +package cache + +import ( + "context" + + "github.com/containerd/containerd/mount" + ocispecs "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" +) + +func (sr *immutableRef) tryComputeOverlayBlob(ctx context.Context, lower, upper []mount.Mount, mediaType string, ref string, compressorFunc compressor) (_ ocispecs.Descriptor, ok bool, err error) { + return ocispecs.Descriptor{}, true, errors.Errorf("overlayfs-based diff computing is unsupported") +} diff --git a/util/testutil/integration/sandbox.go b/util/testutil/integration/sandbox.go index 8367f92889a4f..5e05f9e09e38e 100644 --- a/util/testutil/integration/sandbox.go +++ b/util/testutil/integration/sandbox.go @@ -181,7 +181,7 @@ func runBuildkitd(ctx context.Context, conf *BackendConfig, args []string, logs args = append(args, "--root", tmpdir, "--addr", address, "--debug") cmd := exec.Command(args[0], args[1:]...) - cmd.Env = append(os.Environ(), "BUILDKIT_DEBUG_EXEC_OUTPUT=1", "BUILDKIT_DEBUG_PANIC_ON_ERROR=1", "TMPDIR="+filepath.Join(tmpdir, "tmp")) + cmd.Env = append(os.Environ(), "BUILDKIT_DEBUG_EXEC_OUTPUT=1", "BUILDKIT_DEBUG_PANIC_ON_ERROR=1", "TMPDIR="+filepath.Join(tmpdir, "tmp"), "BUILDKIT_DEBUG_FORCE_OVERLAY_DIFF=1") cmd.Env = append(cmd.Env, extraEnv...) cmd.SysProcAttr = getSysProcAttr()