From 30e8fa4f395daa6bb390980ee6f9e35a5c2d872c Mon Sep 17 00:00:00 2001 From: Daniel J Walsh Date: Thu, 7 Sep 2017 12:11:30 +0000 Subject: [PATCH] Grab changes form moby/moby/graphdriver for overlay quota support This led to lots of packages being updated from moby I tried not to pull in the snap driver stuff. Signed-off-by: Daniel J Walsh --- drivers/aufs/aufs.go | 227 ++++++++++++------ drivers/aufs/aufs_test.go | 117 ++++++---- drivers/aufs/dirs.go | 2 +- drivers/aufs/mount.go | 4 +- drivers/aufs/mount_linux.go | 4 +- drivers/aufs/mount_unsupported.go | 2 +- drivers/btrfs/btrfs.go | 264 ++++++++++++++++----- drivers/btrfs/btrfs_test.go | 4 +- drivers/btrfs/version_test.go | 2 +- drivers/devmapper/deviceset.go | 4 +- drivers/devmapper/driver.go | 8 +- drivers/driver.go | 24 +- drivers/fsdiff.go | 9 +- drivers/graphtest/graphbench_unix.go | 20 +- drivers/graphtest/graphtest_unix.go | 18 +- drivers/graphtest/testutil_unix.go | 2 +- drivers/overlay/mount.go | 8 +- drivers/overlay/overlay.go | 218 ++++++++++++++--- drivers/overlay/overlay_test.go | 21 +- drivers/overlay/randomid.go | 3 +- drivers/overlayutils/overlayutils.go | 18 ++ drivers/proxy.go | 8 +- drivers/quota/projectquota.go | 337 +++++++++++++++++++++++++++ drivers/vfs/driver.go | 49 ++-- drivers/vfs/vfs_test.go | 2 +- drivers/windows/windows.go | 8 +- drivers/zfs/zfs.go | 42 ++-- drivers/zfs/zfs_freebsd.go | 11 +- drivers/zfs/zfs_linux.go | 9 +- drivers/zfs/zfs_solaris.go | 9 +- drivers/zfs/zfs_test.go | 2 +- layers.go | 16 +- store.go | 8 +- 33 files changed, 1128 insertions(+), 352 deletions(-) create mode 100644 drivers/overlayutils/overlayutils.go create mode 100644 drivers/quota/projectquota.go diff --git a/drivers/aufs/aufs.go b/drivers/aufs/aufs.go index 2e7f1e6599..16bfb3da0c 100644 --- a/drivers/aufs/aufs.go +++ b/drivers/aufs/aufs.go @@ -25,6 +25,7 @@ package aufs import ( "bufio" "fmt" + "io" "io/ioutil" "os" "os/exec" @@ -32,22 +33,22 @@ import ( "path/filepath" "strings" "sync" - "syscall" - - "github.com/sirupsen/logrus" - "github.com/vbatts/tar-split/tar/storage" + "time" "github.com/containers/storage/drivers" "github.com/containers/storage/pkg/archive" "github.com/containers/storage/pkg/chrootarchive" "github.com/containers/storage/pkg/directory" "github.com/containers/storage/pkg/idtools" + "github.com/containers/storage/pkg/locker" mountpk "github.com/containers/storage/pkg/mount" - "github.com/containers/storage/pkg/stringid" - + "github.com/containers/storage/pkg/system" rsystem "github.com/opencontainers/runc/libcontainer/system" "github.com/opencontainers/selinux/go-selinux/label" "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "github.com/vbatts/tar-split/tar/storage" + "golang.org/x/sys/unix" ) var ( @@ -74,6 +75,7 @@ type Driver struct { ctr *graphdriver.RefCounter pathCacheLock sync.Mutex pathCache map[string]string + locker *locker.Locker } // Init returns a new AUFS driver. @@ -111,6 +113,7 @@ func Init(root string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap gidMaps: gidMaps, pathCache: make(map[string]string), ctr: graphdriver.NewRefCounter(graphdriver.NewFsChecker(graphdriver.FsMagicAufs)), + locker: locker.New(), } rootUID, rootGID, err := idtools.GetRootUIDGID(uidMaps, gidMaps) @@ -137,6 +140,31 @@ func Init(root string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap return nil, err } } + logger := logrus.WithFields(logrus.Fields{ + "module": "graphdriver", + "driver": "aufs", + }) + + for _, path := range []string{"mnt", "diff"} { + p := filepath.Join(root, path) + entries, err := ioutil.ReadDir(p) + if err != nil { + logger.WithError(err).WithField("dir", p).Error("error reading dir entries") + continue + } + for _, entry := range entries { + if !entry.IsDir() { + continue + } + if strings.HasSuffix(entry.Name(), "-removing") { + logger.WithField("dir", entry.Name()).Debug("Cleaning up stale layer dir") + if err := system.EnsureRemoveAll(filepath.Join(p, entry.Name())); err != nil { + logger.WithField("dir", entry.Name()).WithError(err).Error("Error removing stale layer dir") + } + } + } + } + return a, nil } @@ -200,17 +228,23 @@ func (a *Driver) Exists(id string) bool { return true } +// AdditionalImageStores returns additional image stores supported by the driver +func (a *Driver) AdditionalImageStores() []string { + var imageStores []string + return imageStores +} + // CreateReadWrite creates a layer that is writable for use as a container // file system. -func (a *Driver) CreateReadWrite(id, parent, mountLabel string, storageOpt map[string]string) error { - return a.Create(id, parent, mountLabel, storageOpt) +func (a *Driver) CreateReadWrite(id, parent string, opts *graphdriver.CreateOpts) error { + return a.Create(id, parent, opts) } // Create three folders for each id // mnt, layers, and diff -func (a *Driver) Create(id, parent, mountLabel string, storageOpt map[string]string) error { +func (a *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) error { - if len(storageOpt) != 0 { + if opts != nil && len(opts.StorageOpt) != 0 { return fmt.Errorf("--storage-opt is not supported for aufs") } @@ -225,7 +259,7 @@ func (a *Driver) Create(id, parent, mountLabel string, storageOpt map[string]str defer f.Close() if parent != "" { - ids, err := getParentIds(a.rootPath(), parent) + ids, err := getParentIDs(a.rootPath(), parent) if err != nil { return err } @@ -268,35 +302,68 @@ func (a *Driver) createDirsFor(id string) error { // Remove will unmount and remove the given id. func (a *Driver) Remove(id string) error { + a.locker.Lock(id) + defer a.locker.Unlock(id) a.pathCacheLock.Lock() mountpoint, exists := a.pathCache[id] a.pathCacheLock.Unlock() if !exists { mountpoint = a.getMountpoint(id) } - if err := a.unmount(mountpoint); err != nil { - // no need to return here, we can still try to remove since the `Rename` will fail below if still mounted - logrus.Debugf("aufs: error while unmounting %s: %v", mountpoint, err) - } - // Atomically remove each directory in turn by first moving it out of the - // way (so that container runtimes don't find it anymore) before doing removal of - // the whole tree. - tmpMntPath := path.Join(a.mntPath(), fmt.Sprintf("%s-removing", id)) - if err := os.Rename(mountpoint, tmpMntPath); err != nil && !os.IsNotExist(err) { - return err - } - defer os.RemoveAll(tmpMntPath) + logger := logrus.WithFields(logrus.Fields{ + "module": "graphdriver", + "driver": "aufs", + "layer": id, + }) - tmpDiffpath := path.Join(a.diffPath(), fmt.Sprintf("%s-removing", id)) - if err := os.Rename(a.getDiffPath(id), tmpDiffpath); err != nil && !os.IsNotExist(err) { - return err + var retries int + for { + mounted, err := a.mounted(mountpoint) + if err != nil { + if os.IsNotExist(err) { + break + } + return err + } + if !mounted { + break + } + + err = a.unmount(mountpoint) + if err == nil { + break + } + + if err != unix.EBUSY { + return errors.Wrapf(err, "aufs: unmount error: %s", mountpoint) + } + if retries >= 5 { + return errors.Wrapf(err, "aufs: unmount error after retries: %s", mountpoint) + } + // If unmount returns EBUSY, it could be a transient error. Sleep and retry. + retries++ + logger.Warnf("unmount failed due to EBUSY: retry count: %d", retries) + time.Sleep(100 * time.Millisecond) } - defer os.RemoveAll(tmpDiffpath) // Remove the layers file for the id if err := os.Remove(path.Join(a.rootPath(), "layers", id)); err != nil && !os.IsNotExist(err) { - return err + return errors.Wrapf(err, "error removing layers dir for %s", id) + } + + if err := atomicRemove(a.getDiffPath(id)); err != nil { + return errors.Wrapf(err, "could not remove diff path for id %s", id) + } + + // Atomically remove each directory in turn by first moving it out of the + // way (so that docker doesn't find it anymore) before doing removal of + // the whole tree. + if err := atomicRemove(mountpoint); err != nil { + if errors.Cause(err) == unix.EBUSY { + logger.WithField("dir", mountpoint).WithError(err).Warn("error performing atomic remove due to EBUSY") + } + return errors.Wrapf(err, "could not remove mountpoint for id %s", id) } a.pathCacheLock.Lock() @@ -305,9 +372,29 @@ func (a *Driver) Remove(id string) error { return nil } +func atomicRemove(source string) error { + target := source + "-removing" + + err := os.Rename(source, target) + switch { + case err == nil, os.IsNotExist(err): + case os.IsExist(err): + // Got error saying the target dir already exists, maybe the source doesn't exist due to a previous (failed) remove + if _, e := os.Stat(source); !os.IsNotExist(e) { + return errors.Wrapf(err, "target rename dir '%s' exists but should not, this needs to be manually cleaned up") + } + default: + return errors.Wrapf(err, "error preparing atomic delete") + } + + return system.EnsureRemoveAll(target) +} + // Get returns the rootfs path for the id. -// This will mount the dir at it's given path +// This will mount the dir at its given path func (a *Driver) Get(id, mountLabel string) (string, error) { + a.locker.Lock(id) + defer a.locker.Unlock(id) parents, err := a.getParentLayerPaths(id) if err != nil && !os.IsNotExist(err) { return "", err @@ -343,6 +430,8 @@ func (a *Driver) Get(id, mountLabel string) (string, error) { // Put unmounts and updates list of active mounts. func (a *Driver) Put(id string) error { + a.locker.Lock(id) + defer a.locker.Unlock(id) a.pathCacheLock.Lock() m, exists := a.pathCache[id] if !exists { @@ -363,7 +452,8 @@ func (a *Driver) Put(id string) error { // Diff produces an archive of the changes between the specified // layer and its parent layer which may be "". -func (a *Driver) Diff(id, parent string) (archive.Archive, error) { +func (a *Driver) Diff(id, parent string) (io.ReadCloser, error) { + // AUFS doesn't need the parent layer to produce a diff. return archive.TarWithOptions(path.Join(a.rootPath(), "diff", id), &archive.TarOptions{ Compression: archive.Uncompressed, @@ -373,12 +463,6 @@ func (a *Driver) Diff(id, parent string) (archive.Archive, error) { }) } -// AdditionalImageStores returns additional image stores supported by the driver -func (a *Driver) AdditionalImageStores() []string { - var imageStores []string - return imageStores -} - type fileGetNilCloser struct { storage.FileGetter } @@ -394,7 +478,7 @@ func (a *Driver) DiffGetter(id string) (graphdriver.FileGetCloser, error) { return fileGetNilCloser{storage.NewPathFileGetter(p)}, nil } -func (a *Driver) applyDiff(id string, diff archive.Reader) error { +func (a *Driver) applyDiff(id string, diff io.Reader) error { return chrootarchive.UntarUncompressed(diff, path.Join(a.rootPath(), "diff", id), &archive.TarOptions{ UIDMaps: a.uidMaps, GIDMaps: a.gidMaps, @@ -412,8 +496,9 @@ func (a *Driver) DiffSize(id, parent string) (size int64, err error) { // ApplyDiff extracts the changeset from the given diff into the // layer with the specified id and parent, returning the size of the // new layer in bytes. -func (a *Driver) ApplyDiff(id, parent string, diff archive.Reader) (size int64, err error) { - // AUFS doesn't need the parent id to apply the diff. +func (a *Driver) ApplyDiff(id, parent string, diff io.Reader) (size int64, err error) { + + // AUFS doesn't need the parent id to apply the diff if it is the direct parent. if err = a.applyDiff(id, diff); err != nil { return } @@ -424,6 +509,7 @@ func (a *Driver) ApplyDiff(id, parent string, diff archive.Reader) (size int64, // Changes produces a list of changes between the specified layer // and its parent layer. If parent is "", then all changes will be ADD changes. func (a *Driver) Changes(id, parent string) ([]archive.Change, error) { + // AUFS doesn't have snapshots, so we need to get changes from all parent // layers. layers, err := a.getParentLayerPaths(id) @@ -434,7 +520,7 @@ func (a *Driver) Changes(id, parent string) ([]archive.Change, error) { } func (a *Driver) getParentLayerPaths(id string) ([]string, error) { - parentIds, err := getParentIds(a.rootPath(), id) + parentIds, err := getParentIDs(a.rootPath(), id) if err != nil { return nil, err } @@ -499,7 +585,7 @@ func (a *Driver) Cleanup() error { for _, m := range dirs { if err := a.unmount(m); err != nil { - logrus.Debugf("aufs error unmounting %s: %s", stringid.TruncateID(m), err) + logrus.Debugf("aufs error unmounting %s: %s", m, err) } } return mountpk.Unmount(a.root) @@ -517,45 +603,34 @@ func (a *Driver) aufsMount(ro []string, rw, target, mountLabel string) (err erro offset := 54 if useDirperm() { - offset += len("dirperm1") + offset += len(",dirperm1") } - b := make([]byte, syscall.Getpagesize()-len(mountLabel)-offset) // room for xino & mountLabel + b := make([]byte, unix.Getpagesize()-len(mountLabel)-offset) // room for xino & mountLabel bp := copy(b, fmt.Sprintf("br:%s=rw", rw)) - firstMount := true - i := 0 - - for { - for ; i < len(ro); i++ { - layer := fmt.Sprintf(":%s=ro+wh", ro[i]) - - if firstMount { - if bp+len(layer) > len(b) { - break - } - bp += copy(b[bp:], layer) - } else { - data := label.FormatMountLabel(fmt.Sprintf("append%s", layer), mountLabel) - if err = mount("none", target, "aufs", syscall.MS_REMOUNT, data); err != nil { - return - } - } + index := 0 + for ; index < len(ro); index++ { + layer := fmt.Sprintf(":%s=ro+wh", ro[index]) + if bp+len(layer) > len(b) { + break } + bp += copy(b[bp:], layer) + } - if firstMount { - opts := "dio,xino=/dev/shm/aufs.xino" - if useDirperm() { - opts += ",dirperm1" - } - data := label.FormatMountLabel(fmt.Sprintf("%s,%s", string(b[:bp]), opts), mountLabel) - if err = mount("none", target, "aufs", 0, data); err != nil { - return - } - firstMount = false - } + opts := "dio,xino=/dev/shm/aufs.xino" + if useDirperm() { + opts += ",dirperm1" + } + data := label.FormatMountLabel(fmt.Sprintf("%s,%s", string(b[:bp]), opts), mountLabel) + if err = mount("none", target, "aufs", 0, data); err != nil { + return + } - if i == len(ro) { - break + for ; index < len(ro); index++ { + layer := fmt.Sprintf(":%s=ro+wh", ro[index]) + data := label.FormatMountLabel(fmt.Sprintf("append%s", layer), mountLabel) + if err = mount("none", target, "aufs", unix.MS_REMOUNT, data); err != nil { + return } } @@ -566,14 +641,14 @@ func (a *Driver) aufsMount(ro []string, rw, target, mountLabel string) (err erro // version of aufs. func useDirperm() bool { enableDirpermLock.Do(func() { - base, err := ioutil.TempDir("", "storage-aufs-base") + base, err := ioutil.TempDir("", "docker-aufs-base") if err != nil { logrus.Errorf("error checking dirperm1: %v", err) return } defer os.RemoveAll(base) - union, err := ioutil.TempDir("", "storage-aufs-union") + union, err := ioutil.TempDir("", "docker-aufs-union") if err != nil { logrus.Errorf("error checking dirperm1: %v", err) return diff --git a/drivers/aufs/aufs_test.go b/drivers/aufs/aufs_test.go index b67b4e3da8..bde7d9c777 100644 --- a/drivers/aufs/aufs_test.go +++ b/drivers/aufs/aufs_test.go @@ -12,11 +12,12 @@ import ( "sync" "testing" - "github.com/containers/storage/drivers" + "path/filepath" + + "github.com/containers/storage/daemon/graphdriver" "github.com/containers/storage/pkg/archive" "github.com/containers/storage/pkg/reexec" "github.com/containers/storage/pkg/stringid" - "github.com/pkg/errors" ) var ( @@ -31,7 +32,7 @@ func init() { func testInit(dir string, t testing.TB) graphdriver.Driver { d, err := Init(dir, nil, nil, nil) if err != nil { - if errors.Cause(err) == graphdriver.ErrNotSupported { + if err == graphdriver.ErrNotSupported { t.Skip(err) } else { t.Fatal(err) @@ -57,7 +58,7 @@ func TestNewDriver(t *testing.T) { d := testInit(tmp, t) defer os.RemoveAll(tmp) if d == nil { - t.Fatalf("Driver should not be nil") + t.Fatal("Driver should not be nil") } } @@ -102,7 +103,7 @@ func TestCreateNewDir(t *testing.T) { d := newDriver(t) defer os.RemoveAll(tmp) - if err := d.Create("1", "", "", nil); err != nil { + if err := d.Create("1", "", nil); err != nil { t.Fatal(err) } } @@ -111,7 +112,7 @@ func TestCreateNewDirStructure(t *testing.T) { d := newDriver(t) defer os.RemoveAll(tmp) - if err := d.Create("1", "", "", nil); err != nil { + if err := d.Create("1", "", nil); err != nil { t.Fatal(err) } @@ -132,7 +133,7 @@ func TestRemoveImage(t *testing.T) { d := newDriver(t) defer os.RemoveAll(tmp) - if err := d.Create("1", "", "", nil); err != nil { + if err := d.Create("1", "", nil); err != nil { t.Fatal(err) } @@ -148,7 +149,10 @@ func TestRemoveImage(t *testing.T) { for _, p := range paths { if _, err := os.Stat(path.Join(tmp, p, "1")); err == nil { - t.Fatalf("Error should not be nil because dirs with id 1 should be delted: %s", p) + t.Fatalf("Error should not be nil because dirs with id 1 should be deleted: %s", p) + } + if _, err := os.Stat(path.Join(tmp, p, "1-removing")); err == nil { + t.Fatalf("Error should not be nil because dirs with id 1-removing should be deleted: %s", p) } } } @@ -157,7 +161,7 @@ func TestGetWithoutParent(t *testing.T) { d := newDriver(t) defer os.RemoveAll(tmp) - if err := d.Create("1", "", "", nil); err != nil { + if err := d.Create("1", "", nil); err != nil { t.Fatal(err) } @@ -184,7 +188,7 @@ func TestCleanupWithDir(t *testing.T) { d := newDriver(t) defer os.RemoveAll(tmp) - if err := d.Create("1", "", "", nil); err != nil { + if err := d.Create("1", "", nil); err != nil { t.Fatal(err) } @@ -197,7 +201,7 @@ func TestMountedFalseResponse(t *testing.T) { d := newDriver(t) defer os.RemoveAll(tmp) - if err := d.Create("1", "", "", nil); err != nil { + if err := d.Create("1", "", nil); err != nil { t.Fatal(err) } @@ -207,19 +211,19 @@ func TestMountedFalseResponse(t *testing.T) { } if response != false { - t.Fatalf("Response if dir id 1 is mounted should be false") + t.Fatal("Response if dir id 1 is mounted should be false") } } -func TestMountedTrueReponse(t *testing.T) { +func TestMountedTrueResponse(t *testing.T) { d := newDriver(t) defer os.RemoveAll(tmp) defer d.Cleanup() - if err := d.Create("1", "", "", nil); err != nil { + if err := d.Create("1", "", nil); err != nil { t.Fatal(err) } - if err := d.Create("2", "1", "", nil); err != nil { + if err := d.Create("2", "1", nil); err != nil { t.Fatal(err) } @@ -234,7 +238,7 @@ func TestMountedTrueReponse(t *testing.T) { } if response != true { - t.Fatalf("Response if dir id 2 is mounted should be true") + t.Fatal("Response if dir id 2 is mounted should be true") } } @@ -242,10 +246,10 @@ func TestMountWithParent(t *testing.T) { d := newDriver(t) defer os.RemoveAll(tmp) - if err := d.Create("1", "", "", nil); err != nil { + if err := d.Create("1", "", nil); err != nil { t.Fatal(err) } - if err := d.Create("2", "1", "", nil); err != nil { + if err := d.Create("2", "1", nil); err != nil { t.Fatal(err) } @@ -273,10 +277,10 @@ func TestRemoveMountedDir(t *testing.T) { d := newDriver(t) defer os.RemoveAll(tmp) - if err := d.Create("1", "", "", nil); err != nil { + if err := d.Create("1", "", nil); err != nil { t.Fatal(err) } - if err := d.Create("2", "1", "", nil); err != nil { + if err := d.Create("2", "1", nil); err != nil { t.Fatal(err) } @@ -300,7 +304,7 @@ func TestRemoveMountedDir(t *testing.T) { } if !mounted { - t.Fatalf("Dir id 2 should be mounted") + t.Fatal("Dir id 2 should be mounted") } if err := d.Remove("2"); err != nil { @@ -312,8 +316,8 @@ func TestCreateWithInvalidParent(t *testing.T) { d := newDriver(t) defer os.RemoveAll(tmp) - if err := d.Create("1", "storage", "", nil); err == nil { - t.Fatalf("Error should not be nil with parent does not exist") + if err := d.Create("1", "docker", nil); err == nil { + t.Fatal("Error should not be nil with parent does not exist") } } @@ -321,7 +325,7 @@ func TestGetDiff(t *testing.T) { d := newDriver(t) defer os.RemoveAll(tmp) - if err := d.CreateReadWrite("1", "", "", nil); err != nil { + if err := d.CreateReadWrite("1", "", nil); err != nil { t.Fatal(err) } @@ -347,7 +351,7 @@ func TestGetDiff(t *testing.T) { t.Fatal(err) } if a == nil { - t.Fatalf("Archive should not be nil") + t.Fatal("Archive should not be nil") } } @@ -355,10 +359,11 @@ func TestChanges(t *testing.T) { d := newDriver(t) defer os.RemoveAll(tmp) - if err := d.Create("1", "", "", nil); err != nil { + if err := d.Create("1", "", nil); err != nil { t.Fatal(err) } - if err := d.CreateReadWrite("2", "1", "", nil); err != nil { + + if err := d.CreateReadWrite("2", "1", nil); err != nil { t.Fatal(err) } @@ -404,7 +409,7 @@ func TestChanges(t *testing.T) { t.Fatalf("Change kind should be ChangeAdd got %s", change.Kind) } - if err := d.CreateReadWrite("3", "2", "", nil); err != nil { + if err := d.CreateReadWrite("3", "2", nil); err != nil { t.Fatal(err) } mntPoint, err = d.Get("3", "") @@ -425,7 +430,7 @@ func TestChanges(t *testing.T) { t.Fatal(err) } - changes, err = d.Changes("3", "") + changes, err = d.Changes("3", "2") if err != nil { t.Fatal(err) } @@ -449,7 +454,7 @@ func TestDiffSize(t *testing.T) { d := newDriver(t) defer os.RemoveAll(tmp) - if err := d.CreateReadWrite("1", "", "", nil); err != nil { + if err := d.CreateReadWrite("1", "", nil); err != nil { t.Fatal(err) } @@ -491,7 +496,7 @@ func TestChildDiffSize(t *testing.T) { defer os.RemoveAll(tmp) defer d.Cleanup() - if err := d.CreateReadWrite("1", "", "", nil); err != nil { + if err := d.CreateReadWrite("1", "", nil); err != nil { t.Fatal(err) } @@ -527,11 +532,11 @@ func TestChildDiffSize(t *testing.T) { t.Fatalf("Expected size to be %d got %d", size, diffSize) } - if err := d.Create("2", "1", "", nil); err != nil { + if err := d.Create("2", "1", nil); err != nil { t.Fatal(err) } - diffSize, err = d.DiffSize("2", "") + diffSize, err = d.DiffSize("2", "1") if err != nil { t.Fatal(err) } @@ -546,12 +551,12 @@ func TestExists(t *testing.T) { defer os.RemoveAll(tmp) defer d.Cleanup() - if err := d.Create("1", "", "", nil); err != nil { + if err := d.Create("1", "", nil); err != nil { t.Fatal(err) } if d.Exists("none") { - t.Fatal("id name should not exist in the driver") + t.Fatal("id none should not exist in the driver") } if !d.Exists("1") { @@ -564,7 +569,7 @@ func TestStatus(t *testing.T) { defer os.RemoveAll(tmp) defer d.Cleanup() - if err := d.Create("1", "", "", nil); err != nil { + if err := d.Create("1", "", nil); err != nil { t.Fatal(err) } @@ -593,7 +598,7 @@ func TestApplyDiff(t *testing.T) { defer os.RemoveAll(tmp) defer d.Cleanup() - if err := d.CreateReadWrite("1", "", "", nil); err != nil { + if err := d.CreateReadWrite("1", "", nil); err != nil { t.Fatal(err) } @@ -619,10 +624,10 @@ func TestApplyDiff(t *testing.T) { t.Fatal(err) } - if err := d.Create("2", "", "", nil); err != nil { + if err := d.Create("2", "", nil); err != nil { t.Fatal(err) } - if err := d.Create("3", "2", "", nil); err != nil { + if err := d.Create("3", "2", nil); err != nil { t.Fatal(err) } @@ -672,7 +677,7 @@ func testMountMoreThan42Layers(t *testing.T, mountPath string) { } current = hash(current) - if err := d.CreateReadWrite(current, parent, "", nil); err != nil { + if err := d.CreateReadWrite(current, parent, nil); err != nil { t.Logf("Current layer %d", i) t.Error(err) } @@ -744,25 +749,25 @@ func BenchmarkConcurrentAccess(b *testing.B) { defer os.RemoveAll(tmp) defer d.Cleanup() - numConcurent := 256 + numConcurrent := 256 // create a bunch of ids var ids []string - for i := 0; i < numConcurent; i++ { + for i := 0; i < numConcurrent; i++ { ids = append(ids, stringid.GenerateNonCryptoID()) } - if err := d.Create(ids[0], "", "", nil); err != nil { + if err := d.Create(ids[0], "", nil); err != nil { b.Fatal(err) } - if err := d.Create(ids[1], ids[0], "", nil); err != nil { + if err := d.Create(ids[1], ids[0], nil); err != nil { b.Fatal(err) } parent := ids[1] ids = append(ids[2:]) - chErr := make(chan error, numConcurent) + chErr := make(chan error, numConcurrent) var outerGroup sync.WaitGroup outerGroup.Add(len(ids)) b.StartTimer() @@ -771,7 +776,7 @@ func BenchmarkConcurrentAccess(b *testing.B) { for _, id := range ids { go func(id string) { defer outerGroup.Done() - if err := d.Create(id, parent, "", nil); err != nil { + if err := d.Create(id, parent, nil); err != nil { b.Logf("Create %s failed", id) chErr <- err return @@ -800,3 +805,23 @@ func BenchmarkConcurrentAccess(b *testing.B) { } } } + +func TestInitStaleCleanup(t *testing.T) { + if err := os.MkdirAll(tmp, 0755); err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmp) + + for _, d := range []string{"diff", "mnt"} { + if err := os.MkdirAll(filepath.Join(tmp, d, "123-removing"), 0755); err != nil { + t.Fatal(err) + } + } + + testInit(tmp, t) + for _, d := range []string{"diff", "mnt"} { + if _, err := os.Stat(filepath.Join(tmp, d, "123-removing")); err == nil { + t.Fatal("cleanup failed") + } + } +} diff --git a/drivers/aufs/dirs.go b/drivers/aufs/dirs.go index eb298d9eeb..d2325fc46c 100644 --- a/drivers/aufs/dirs.go +++ b/drivers/aufs/dirs.go @@ -29,7 +29,7 @@ func loadIds(root string) ([]string, error) { // // If there are no lines in the file then the id has no parent // and an empty slice is returned. -func getParentIds(root, id string) ([]string, error) { +func getParentIDs(root, id string) ([]string, error) { f, err := os.Open(path.Join(root, "layers", id)) if err != nil { return nil, err diff --git a/drivers/aufs/mount.go b/drivers/aufs/mount.go index 8314f142bd..100e7537a9 100644 --- a/drivers/aufs/mount.go +++ b/drivers/aufs/mount.go @@ -4,9 +4,9 @@ package aufs import ( "os/exec" - "syscall" "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" ) // Unmount the target specified. @@ -14,7 +14,7 @@ func Unmount(target string) error { if err := exec.Command("auplink", target, "flush").Run(); err != nil { logrus.Warnf("Couldn't run auplink before unmount %s: %s", target, err) } - if err := syscall.Unmount(target, 0); err != nil { + if err := unix.Unmount(target, 0); err != nil { return err } return nil diff --git a/drivers/aufs/mount_linux.go b/drivers/aufs/mount_linux.go index 8062bae420..937104ba3f 100644 --- a/drivers/aufs/mount_linux.go +++ b/drivers/aufs/mount_linux.go @@ -1,7 +1,7 @@ package aufs -import "syscall" +import "golang.org/x/sys/unix" func mount(source string, target string, fstype string, flags uintptr, data string) error { - return syscall.Mount(source, target, fstype, flags, data) + return unix.Mount(source, target, fstype, flags, data) } diff --git a/drivers/aufs/mount_unsupported.go b/drivers/aufs/mount_unsupported.go index c807902df9..d030b06637 100644 --- a/drivers/aufs/mount_unsupported.go +++ b/drivers/aufs/mount_unsupported.go @@ -2,7 +2,7 @@ package aufs -import "github.com/pkg/errors" +import "errors" // MsRemount declared to specify a non-linux system mount. const MsRemount = 0 diff --git a/drivers/btrfs/btrfs.go b/drivers/btrfs/btrfs.go index 9e16f89457..52c55588a2 100644 --- a/drivers/btrfs/btrfs.go +++ b/drivers/btrfs/btrfs.go @@ -16,31 +16,31 @@ import "C" import ( "fmt" + "io/ioutil" + "math" "os" "path" "path/filepath" + "strconv" "strings" - "syscall" + "sync" "unsafe" "github.com/containers/storage/drivers" "github.com/containers/storage/pkg/idtools" "github.com/containers/storage/pkg/mount" "github.com/containers/storage/pkg/parsers" + "github.com/containers/storage/pkg/system" "github.com/docker/go-units" "github.com/opencontainers/selinux/go-selinux/label" - "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" ) func init() { graphdriver.Register("btrfs", Init) } -var ( - quotaEnabled = false - userDiskQuota = false -) - type btrfsOptions struct { minSpace uint64 size uint64 @@ -56,7 +56,7 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap } if fsMagic != graphdriver.FsMagicBtrfs { - return nil, errors.Wrapf(graphdriver.ErrPrerequisites, "%q is not on a btrfs filesystem", home) + return nil, graphdriver.ErrPrerequisites } rootUID, rootGID, err := idtools.GetRootUIDGID(uidMaps, gidMaps) @@ -71,18 +71,11 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap return nil, err } - opt, err := parseOptions(options) + opt, userDiskQuota, err := parseOptions(options) if err != nil { return nil, err } - if userDiskQuota { - if err := subvolEnableQuota(home); err != nil { - return nil, err - } - quotaEnabled = true - } - driver := &Driver{ home: home, uidMaps: uidMaps, @@ -90,39 +83,48 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap options: opt, } + if userDiskQuota { + if err := driver.subvolEnableQuota(); err != nil { + return nil, err + } + } + return graphdriver.NewNaiveDiffDriver(driver, uidMaps, gidMaps), nil } -func parseOptions(opt []string) (btrfsOptions, error) { +func parseOptions(opt []string) (btrfsOptions, bool, error) { var options btrfsOptions + userDiskQuota := false for _, option := range opt { key, val, err := parsers.ParseKeyValueOpt(option) if err != nil { - return options, err + return options, userDiskQuota, err } key = strings.ToLower(key) switch key { case "btrfs.min_space": minSpace, err := units.RAMInBytes(val) if err != nil { - return options, err + return options, userDiskQuota, err } userDiskQuota = true options.minSpace = uint64(minSpace) default: - return options, fmt.Errorf("Unknown option %s", key) + return options, userDiskQuota, fmt.Errorf("Unknown option %s", key) } } - return options, nil + return options, userDiskQuota, nil } // Driver contains information about the filesystem mounted. type Driver struct { //root of the file system - home string - uidMaps []idtools.IDMap - gidMaps []idtools.IDMap - options btrfsOptions + home string + uidMaps []idtools.IDMap + gidMaps []idtools.IDMap + options btrfsOptions + quotaEnabled bool + once sync.Once } // String prints the name of the driver (btrfs). @@ -151,10 +153,8 @@ func (d *Driver) Metadata(id string) (map[string]string, error) { // Cleanup unmounts the home directory. func (d *Driver) Cleanup() error { - if quotaEnabled { - if err := subvolDisableQuota(d.home); err != nil { - return err - } + if err := d.subvolDisableQuota(); err != nil { + return err } return mount.Unmount(d.home) @@ -197,7 +197,7 @@ func subvolCreate(path, name string) error { args.name[i] = C.char(c) } - _, _, errno := syscall.Syscall(syscall.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_SUBVOL_CREATE, + _, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_SUBVOL_CREATE, uintptr(unsafe.Pointer(&args))) if errno != 0 { return fmt.Errorf("Failed to create btrfs subvolume: %v", errno.Error()) @@ -225,7 +225,7 @@ func subvolSnapshot(src, dest, name string) error { C.set_name_btrfs_ioctl_vol_args_v2(&args, cs) C.free(unsafe.Pointer(cs)) - _, _, errno := syscall.Syscall(syscall.SYS_IOCTL, getDirFd(destDir), C.BTRFS_IOC_SNAP_CREATE_V2, + _, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(destDir), C.BTRFS_IOC_SNAP_CREATE_V2, uintptr(unsafe.Pointer(&args))) if errno != 0 { return fmt.Errorf("Failed to create btrfs snapshot: %v", errno.Error()) @@ -234,8 +234,8 @@ func subvolSnapshot(src, dest, name string) error { } func isSubvolume(p string) (bool, error) { - var bufStat syscall.Stat_t - if err := syscall.Lstat(p, &bufStat); err != nil { + var bufStat unix.Stat_t + if err := unix.Lstat(p, &bufStat); err != nil { return false, err } @@ -243,7 +243,7 @@ func isSubvolume(p string) (bool, error) { return bufStat.Ino == C.BTRFS_FIRST_FREE_OBJECTID, nil } -func subvolDelete(dirpath, name string) error { +func subvolDelete(dirpath, name string, quotaEnabled bool) error { dir, err := openDir(dirpath) if err != nil { return err @@ -271,7 +271,7 @@ func subvolDelete(dirpath, name string) error { return fmt.Errorf("Failed to test if %s is a btrfs subvolume: %v", p, err) } if sv { - if err := subvolDelete(path.Dir(p), f.Name()); err != nil { + if err := subvolDelete(path.Dir(p), f.Name(), quotaEnabled); err != nil { return fmt.Errorf("Failed to destroy btrfs child subvolume (%s) of parent (%s): %v", p, dirpath, err) } } @@ -282,12 +282,27 @@ func subvolDelete(dirpath, name string) error { return fmt.Errorf("Recursively walking subvolumes for %s failed: %v", dirpath, err) } + if quotaEnabled { + if qgroupid, err := subvolLookupQgroup(fullPath); err == nil { + var args C.struct_btrfs_ioctl_qgroup_create_args + args.qgroupid = C.__u64(qgroupid) + + _, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_QGROUP_CREATE, + uintptr(unsafe.Pointer(&args))) + if errno != 0 { + logrus.Errorf("Failed to delete btrfs qgroup %v for %s: %v", qgroupid, fullPath, errno.Error()) + } + } else { + logrus.Errorf("Failed to lookup btrfs qgroup for %s: %v", fullPath, err.Error()) + } + } + // all subvolumes have been removed // now remove the one originally passed in for i, c := range []byte(name) { args.name[i] = C.char(c) } - _, _, errno := syscall.Syscall(syscall.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_SNAP_DESTROY, + _, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_SNAP_DESTROY, uintptr(unsafe.Pointer(&args))) if errno != 0 { return fmt.Errorf("Failed to destroy btrfs snapshot %s for %s: %v", dirpath, name, errno.Error()) @@ -295,8 +310,27 @@ func subvolDelete(dirpath, name string) error { return nil } -func subvolEnableQuota(path string) error { - dir, err := openDir(path) +func (d *Driver) updateQuotaStatus() { + d.once.Do(func() { + if !d.quotaEnabled { + // In case quotaEnabled is not set, check qgroup and update quotaEnabled as needed + if err := subvolQgroupStatus(d.home); err != nil { + // quota is still not enabled + return + } + d.quotaEnabled = true + } + }) +} + +func (d *Driver) subvolEnableQuota() error { + d.updateQuotaStatus() + + if d.quotaEnabled { + return nil + } + + dir, err := openDir(d.home) if err != nil { return err } @@ -304,17 +338,25 @@ func subvolEnableQuota(path string) error { var args C.struct_btrfs_ioctl_quota_ctl_args args.cmd = C.BTRFS_QUOTA_CTL_ENABLE - _, _, errno := syscall.Syscall(syscall.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_QUOTA_CTL, + _, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_QUOTA_CTL, uintptr(unsafe.Pointer(&args))) if errno != 0 { return fmt.Errorf("Failed to enable btrfs quota for %s: %v", dir, errno.Error()) } + d.quotaEnabled = true + return nil } -func subvolDisableQuota(path string) error { - dir, err := openDir(path) +func (d *Driver) subvolDisableQuota() error { + d.updateQuotaStatus() + + if !d.quotaEnabled { + return nil + } + + dir, err := openDir(d.home) if err != nil { return err } @@ -322,24 +364,32 @@ func subvolDisableQuota(path string) error { var args C.struct_btrfs_ioctl_quota_ctl_args args.cmd = C.BTRFS_QUOTA_CTL_DISABLE - _, _, errno := syscall.Syscall(syscall.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_QUOTA_CTL, + _, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_QUOTA_CTL, uintptr(unsafe.Pointer(&args))) if errno != 0 { return fmt.Errorf("Failed to disable btrfs quota for %s: %v", dir, errno.Error()) } + d.quotaEnabled = false + return nil } -func subvolRescanQuota(path string) error { - dir, err := openDir(path) +func (d *Driver) subvolRescanQuota() error { + d.updateQuotaStatus() + + if !d.quotaEnabled { + return nil + } + + dir, err := openDir(d.home) if err != nil { return err } defer closeDir(dir) var args C.struct_btrfs_ioctl_quota_rescan_args - _, _, errno := syscall.Syscall(syscall.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_QUOTA_RESCAN_WAIT, + _, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_QUOTA_RESCAN_WAIT, uintptr(unsafe.Pointer(&args))) if errno != 0 { return fmt.Errorf("Failed to rescan btrfs quota for %s: %v", dir, errno.Error()) @@ -358,7 +408,7 @@ func subvolLimitQgroup(path string, size uint64) error { var args C.struct_btrfs_ioctl_qgroup_limit_args args.lim.max_referenced = C.__u64(size) args.lim.flags = C.BTRFS_QGROUP_LIMIT_MAX_RFER - _, _, errno := syscall.Syscall(syscall.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_QGROUP_LIMIT, + _, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_QGROUP_LIMIT, uintptr(unsafe.Pointer(&args))) if errno != 0 { return fmt.Errorf("Failed to limit qgroup for %s: %v", dir, errno.Error()) @@ -367,6 +417,60 @@ func subvolLimitQgroup(path string, size uint64) error { return nil } +// subvolQgroupStatus performs a BTRFS_IOC_TREE_SEARCH on the root path +// with search key of BTRFS_QGROUP_STATUS_KEY. +// In case qgroup is enabled, the retuned key type will match BTRFS_QGROUP_STATUS_KEY. +// For more details please see https://github.com/kdave/btrfs-progs/blob/v4.9/qgroup.c#L1035 +func subvolQgroupStatus(path string) error { + dir, err := openDir(path) + if err != nil { + return err + } + defer closeDir(dir) + + var args C.struct_btrfs_ioctl_search_args + args.key.tree_id = C.BTRFS_QUOTA_TREE_OBJECTID + args.key.min_type = C.BTRFS_QGROUP_STATUS_KEY + args.key.max_type = C.BTRFS_QGROUP_STATUS_KEY + args.key.max_objectid = C.__u64(math.MaxUint64) + args.key.max_offset = C.__u64(math.MaxUint64) + args.key.max_transid = C.__u64(math.MaxUint64) + args.key.nr_items = 4096 + + _, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_TREE_SEARCH, + uintptr(unsafe.Pointer(&args))) + if errno != 0 { + return fmt.Errorf("Failed to search qgroup for %s: %v", path, errno.Error()) + } + sh := (*C.struct_btrfs_ioctl_search_header)(unsafe.Pointer(&args.buf)) + if sh._type != C.BTRFS_QGROUP_STATUS_KEY { + return fmt.Errorf("Invalid qgroup search header type for %s: %v", path, sh._type) + } + return nil +} + +func subvolLookupQgroup(path string) (uint64, error) { + dir, err := openDir(path) + if err != nil { + return 0, err + } + defer closeDir(dir) + + var args C.struct_btrfs_ioctl_ino_lookup_args + args.objectid = C.BTRFS_FIRST_FREE_OBJECTID + + _, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_INO_LOOKUP, + uintptr(unsafe.Pointer(&args))) + if errno != 0 { + return 0, fmt.Errorf("Failed to lookup qgroup for %s: %v", dir, errno.Error()) + } + if args.treeid == 0 { + return 0, fmt.Errorf("Invalid qgroup id for %s: 0", dir) + } + + return uint64(args.treeid), nil +} + func (d *Driver) subvolumesDir() string { return path.Join(d.home, "subvolumes") } @@ -375,14 +479,23 @@ func (d *Driver) subvolumesDirID(id string) string { return path.Join(d.subvolumesDir(), id) } +func (d *Driver) quotasDir() string { + return path.Join(d.home, "quotas") +} + +func (d *Driver) quotasDirID(id string) string { + return path.Join(d.quotasDir(), id) +} + // CreateReadWrite creates a layer that is writable for use as a container // file system. -func (d *Driver) CreateReadWrite(id, parent, mountLabel string, storageOpt map[string]string) error { - return d.Create(id, parent, mountLabel, storageOpt) +func (d *Driver) CreateReadWrite(id, parent string, opts *graphdriver.CreateOpts) error { + return d.Create(id, parent, opts) } // Create the filesystem with given id. -func (d *Driver) Create(id, parent, mountLabel string, storageOpt map[string]string) error { +func (d *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) error { + quotas := path.Join(d.home, "quotas") subvolumes := path.Join(d.home, "subvolumes") rootUID, rootGID, err := idtools.GetRootUIDGID(d.uidMaps, d.gidMaps) if err != nil { @@ -409,14 +522,26 @@ func (d *Driver) Create(id, parent, mountLabel string, storageOpt map[string]str } } + var storageOpt map[string]string + if opts != nil { + storageOpt = opts.StorageOpt + } + if _, ok := storageOpt["size"]; ok { driver := &Driver{} if err := d.parseStorageOpt(storageOpt, driver); err != nil { return err } + if err := d.setStorageSize(path.Join(subvolumes, id), driver); err != nil { return err } + if err := idtools.MkdirAllAs(quotas, 0700, rootUID, rootGID); err != nil { + return err + } + if err := ioutil.WriteFile(path.Join(quotas, id), []byte(fmt.Sprint(driver.options.size)), 0644); err != nil { + return err + } } // if we have a remapped root (user namespaces enabled), change the created snapshot @@ -427,6 +552,11 @@ func (d *Driver) Create(id, parent, mountLabel string, storageOpt map[string]str } } + mountLabel := "" + if opts != nil { + mountLabel = opts.MountLabel + } + return label.Relabel(path.Join(subvolumes, id), mountLabel, false) } @@ -459,11 +589,8 @@ func (d *Driver) setStorageSize(dir string, driver *Driver) error { return fmt.Errorf("btrfs: storage size cannot be less than %s", units.HumanSize(float64(d.options.minSpace))) } - if !quotaEnabled { - if err := subvolEnableQuota(d.home); err != nil { - return err - } - quotaEnabled = true + if err := d.subvolEnableQuota(); err != nil { + return err } if err := subvolLimitQgroup(dir, driver.options.size); err != nil { @@ -479,13 +606,25 @@ func (d *Driver) Remove(id string) error { if _, err := os.Stat(dir); err != nil { return err } - if err := subvolDelete(d.subvolumesDir(), id); err != nil { + quotasDir := d.quotasDirID(id) + if _, err := os.Stat(quotasDir); err == nil { + if err := os.Remove(quotasDir); err != nil { + return err + } + } else if !os.IsNotExist(err) { + return err + } + + // Call updateQuotaStatus() to invoke status update + d.updateQuotaStatus() + + if err := subvolDelete(d.subvolumesDir(), id, d.quotaEnabled); err != nil { return err } - if err := os.RemoveAll(dir); err != nil && !os.IsNotExist(err) { + if err := system.EnsureRemoveAll(dir); err != nil { return err } - if err := subvolRescanQuota(d.home); err != nil { + if err := d.subvolRescanQuota(); err != nil { return err } return nil @@ -503,6 +642,17 @@ func (d *Driver) Get(id, mountLabel string) (string, error) { return "", fmt.Errorf("%s: not a directory", dir) } + if quota, err := ioutil.ReadFile(d.quotasDirID(id)); err == nil { + if size, err := strconv.ParseUint(string(quota), 10, 64); err == nil && size >= d.options.minSpace { + if err := d.subvolEnableQuota(); err != nil { + return "", err + } + if err := subvolLimitQgroup(dir, size); err != nil { + return "", err + } + } + } + return dir, nil } diff --git a/drivers/btrfs/btrfs_test.go b/drivers/btrfs/btrfs_test.go index b6372f81bc..554f8a55db 100644 --- a/drivers/btrfs/btrfs_test.go +++ b/drivers/btrfs/btrfs_test.go @@ -7,7 +7,7 @@ import ( "path" "testing" - "github.com/containers/storage/drivers/graphtest" + "github.com/containers/storage/daemon/graphdriver/graphtest" ) // This avoids creating a new driver for each test if all tests are run @@ -30,7 +30,7 @@ func TestBtrfsCreateSnap(t *testing.T) { func TestBtrfsSubvolDelete(t *testing.T) { d := graphtest.GetDriver(t, "btrfs") - if err := d.CreateReadWrite("test", "", "", nil); err != nil { + if err := d.CreateReadWrite("test", "", nil); err != nil { t.Fatal(err) } defer graphtest.PutDriver(t) diff --git a/drivers/btrfs/version_test.go b/drivers/btrfs/version_test.go index 15a6e75cb3..d78d577179 100644 --- a/drivers/btrfs/version_test.go +++ b/drivers/btrfs/version_test.go @@ -8,6 +8,6 @@ import ( func TestLibVersion(t *testing.T) { if btrfsLibVersion() <= 0 { - t.Errorf("expected output from btrfs lib version > 0") + t.Error("expected output from btrfs lib version > 0") } } diff --git a/drivers/devmapper/deviceset.go b/drivers/devmapper/deviceset.go index 2608c49f4b..f2b53eb2e2 100644 --- a/drivers/devmapper/deviceset.go +++ b/drivers/devmapper/deviceset.go @@ -1856,7 +1856,7 @@ func (devices *DeviceSet) initDevmapper(doInit bool) error { } // AddDevice adds a device and registers in the hash. -func (devices *DeviceSet) AddDevice(hash, baseHash string, storageOpt map[string]string) error { +func (devices *DeviceSet) AddDevice(hash, baseHash string, opts *graphdriver.CreateOpts) error { logrus.Debugf("devmapper: AddDevice(hash=%s basehash=%s)", hash, baseHash) defer logrus.Debugf("devmapper: AddDevice(hash=%s basehash=%s) END", hash, baseHash) @@ -1882,7 +1882,7 @@ func (devices *DeviceSet) AddDevice(hash, baseHash string, storageOpt map[string return fmt.Errorf("devmapper: device %s already exists. Deleted=%v", hash, info.Deleted) } - size, err := devices.parseStorageOpt(storageOpt) + size, err := devices.parseStorageOpt(opts.StorageOpt) if err != nil { return err } diff --git a/drivers/devmapper/driver.go b/drivers/devmapper/driver.go index 87a427a866..fc07143270 100644 --- a/drivers/devmapper/driver.go +++ b/drivers/devmapper/driver.go @@ -122,13 +122,13 @@ func (d *Driver) Cleanup() error { // CreateReadWrite creates a layer that is writable for use as a container // file system. -func (d *Driver) CreateReadWrite(id, parent, mountLabel string, storageOpt map[string]string) error { - return d.Create(id, parent, mountLabel, storageOpt) +func (d *Driver) CreateReadWrite(id, parent string, opts *graphdriver.CreateOpts) error { + return d.Create(id, parent, opts) } // Create adds a device with a given id and the parent. -func (d *Driver) Create(id, parent, mountLabel string, storageOpt map[string]string) error { - if err := d.DeviceSet.AddDevice(id, parent, storageOpt); err != nil { +func (d *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) error { + if err := d.DeviceSet.AddDevice(id, parent, opts); err != nil { return err } diff --git a/drivers/driver.go b/drivers/driver.go index c16fc33e11..0fb9797328 100644 --- a/drivers/driver.go +++ b/drivers/driver.go @@ -2,6 +2,7 @@ package graphdriver import ( "fmt" + "io" "os" "path/filepath" "strings" @@ -28,12 +29,19 @@ var ( // ErrNotSupported returned when driver is not supported. ErrNotSupported = errors.New("driver not supported") - // ErrPrerequisites retuned when driver does not meet prerequisites. + // ErrPrerequisites returned when driver does not meet prerequisites. ErrPrerequisites = errors.New("prerequisites for driver not satisfied (wrong filesystem?)") // ErrIncompatibleFS returned when file system is not supported. ErrIncompatibleFS = fmt.Errorf("backing file system is unsupported for this graph driver") ) +//CreateOpts contains optional arguments for Create() and CreateReadWrite() +// methods. +type CreateOpts struct { + MountLabel string + StorageOpt map[string]string +} + // InitFunc initializes the storage driver. type InitFunc func(root string, options []string, uidMaps, gidMaps []idtools.IDMap) (Driver, error) @@ -47,11 +55,13 @@ type ProtoDriver interface { // String returns a string representation of this driver. String() string // CreateReadWrite creates a new, empty filesystem layer that is ready - // to be used as the storage for a container. - CreateReadWrite(id, parent, mountLabel string, storageOpt map[string]string) error + // to be used as the storage for a container. Additional options can + // be passed in opts. parent may be "" and opts may be nil. + CreateReadWrite(id, parent string, opts *CreateOpts) error // Create creates a new, empty, filesystem layer with the - // specified id and parent and mountLabel. Parent and mountLabel may be "". - Create(id, parent, mountLabel string, storageOpt map[string]string) error + // specified id and parent and options passed in opts. Parent + // may be "" and opts may be nil. + Create(id, parent string, opts *CreateOpts) error // Remove attempts to remove the filesystem layer with this id. Remove(id string) error // Get returns the mountpoint for the layered filesystem referred @@ -83,7 +93,7 @@ type Driver interface { ProtoDriver // Diff produces an archive of the changes between the specified // layer and its parent layer which may be "". - Diff(id, parent string) (archive.Archive, error) + Diff(id, parent string) (io.ReadCloser, error) // Changes produces a list of changes between the specified layer // and its parent layer. If parent is "", then all changes will be ADD changes. Changes(id, parent string) ([]archive.Change, error) @@ -91,7 +101,7 @@ type Driver interface { // layer with the specified id and parent, returning the size of the // new layer in bytes. // The archive.Reader must be an uncompressed stream. - ApplyDiff(id, parent string, diff archive.Reader) (size int64, err error) + ApplyDiff(id, parent string, diff io.Reader) (size int64, err error) // DiffSize calculates the changes between the specified id // and its parent and returns the size in bytes of the changes // relative to its base filesystem directory. diff --git a/drivers/fsdiff.go b/drivers/fsdiff.go index 693107295e..b521c4763d 100644 --- a/drivers/fsdiff.go +++ b/drivers/fsdiff.go @@ -1,6 +1,7 @@ package graphdriver import ( + "io" "time" "github.com/sirupsen/logrus" @@ -31,9 +32,9 @@ type NaiveDiffDriver struct { // NewNaiveDiffDriver returns a fully functional driver that wraps the // given ProtoDriver and adds the capability of the following methods which // it may or may not support on its own: -// Diff(id, parent string) (archive.Archive, error) +// Diff(id, parent string) (io.ReadCloser, error) // Changes(id, parent string) ([]archive.Change, error) -// ApplyDiff(id, parent string, diff archive.Reader) (size int64, err error) +// ApplyDiff(id, parent string, diff io.Reader) (size int64, err error) // DiffSize(id, parent string) (size int64, err error) func NewNaiveDiffDriver(driver ProtoDriver, uidMaps, gidMaps []idtools.IDMap) Driver { gdw := &NaiveDiffDriver{ @@ -46,7 +47,7 @@ func NewNaiveDiffDriver(driver ProtoDriver, uidMaps, gidMaps []idtools.IDMap) Dr // Diff produces an archive of the changes between the specified // layer and its parent layer which may be "". -func (gdw *NaiveDiffDriver) Diff(id, parent string) (arch archive.Archive, err error) { +func (gdw *NaiveDiffDriver) Diff(id, parent string) (arch io.ReadCloser, err error) { layerFs, err := gdw.Get(id, "") if err != nil { return nil, err @@ -118,7 +119,7 @@ func (gdw *NaiveDiffDriver) Changes(id, parent string) ([]archive.Change, error) // ApplyDiff extracts the changeset from the given diff into the // layer with the specified id and parent, returning the size of the // new layer in bytes. -func (gdw *NaiveDiffDriver) ApplyDiff(id, parent string, diff archive.Reader) (size int64, err error) { +func (gdw *NaiveDiffDriver) ApplyDiff(id, parent string, diff io.Reader) (size int64, err error) { // Mount the root filesystem so we can apply the diff/layer. layerFs, err := gdw.Get(id, "") if err != nil { diff --git a/drivers/graphtest/graphbench_unix.go b/drivers/graphtest/graphbench_unix.go index 5498eac30a..ff36689611 100644 --- a/drivers/graphtest/graphbench_unix.go +++ b/drivers/graphtest/graphbench_unix.go @@ -19,7 +19,7 @@ func DriverBenchExists(b *testing.B, drivername string, driveroptions ...string) base := stringid.GenerateRandomID() - if err := driver.Create(base, "", "", nil); err != nil { + if err := driver.Create(base, "", nil); err != nil { b.Fatal(err) } @@ -38,7 +38,7 @@ func DriverBenchGetEmpty(b *testing.B, drivername string, driveroptions ...strin base := stringid.GenerateRandomID() - if err := driver.Create(base, "", "", nil); err != nil { + if err := driver.Create(base, "", nil); err != nil { b.Fatal(err) } @@ -63,7 +63,7 @@ func DriverBenchDiffBase(b *testing.B, drivername string, driveroptions ...strin base := stringid.GenerateRandomID() - if err := driver.Create(base, "", "", nil); err != nil { + if err := driver.Create(base, "", nil); err != nil { b.Fatal(err) } @@ -93,7 +93,7 @@ func DriverBenchDiffN(b *testing.B, bottom, top int, drivername string, driverop base := stringid.GenerateRandomID() upper := stringid.GenerateRandomID() - if err := driver.Create(base, "", "", nil); err != nil { + if err := driver.Create(base, "", nil); err != nil { b.Fatal(err) } @@ -101,7 +101,7 @@ func DriverBenchDiffN(b *testing.B, bottom, top int, drivername string, driverop b.Fatal(err) } - if err := driver.Create(upper, base, "", nil); err != nil { + if err := driver.Create(upper, base, nil); err != nil { b.Fatal(err) } @@ -129,7 +129,7 @@ func DriverBenchDiffApplyN(b *testing.B, fileCount int, drivername string, drive base := stringid.GenerateRandomID() upper := stringid.GenerateRandomID() - if err := driver.Create(base, "", "", nil); err != nil { + if err := driver.Create(base, "", nil); err != nil { b.Fatal(err) } @@ -137,7 +137,7 @@ func DriverBenchDiffApplyN(b *testing.B, fileCount int, drivername string, drive b.Fatal(err) } - if err := driver.Create(upper, base, "", nil); err != nil { + if err := driver.Create(upper, base, nil); err != nil { b.Fatal(err) } @@ -152,7 +152,7 @@ func DriverBenchDiffApplyN(b *testing.B, fileCount int, drivername string, drive b.StopTimer() for i := 0; i < b.N; i++ { diff := stringid.GenerateRandomID() - if err := driver.Create(diff, base, "", nil); err != nil { + if err := driver.Create(diff, base, nil); err != nil { b.Fatal(err) } @@ -192,7 +192,7 @@ func DriverBenchDeepLayerDiff(b *testing.B, layerCount int, drivername string, d base := stringid.GenerateRandomID() - if err := driver.Create(base, "", "", nil); err != nil { + if err := driver.Create(base, "", nil); err != nil { b.Fatal(err) } @@ -226,7 +226,7 @@ func DriverBenchDeepLayerRead(b *testing.B, layerCount int, drivername string, d base := stringid.GenerateRandomID() - if err := driver.Create(base, "", "", nil); err != nil { + if err := driver.Create(base, "", nil); err != nil { b.Fatal(err) } diff --git a/drivers/graphtest/graphtest_unix.go b/drivers/graphtest/graphtest_unix.go index fbee026f95..1bfcd1b2a5 100644 --- a/drivers/graphtest/graphtest_unix.go +++ b/drivers/graphtest/graphtest_unix.go @@ -88,7 +88,7 @@ func DriverTestCreateEmpty(t testing.TB, drivername string, driverOptions ...str driver := GetDriver(t, drivername, driverOptions...) defer PutDriver(t) - if err := driver.Create("empty", "", "", nil); err != nil { + if err := driver.Create("empty", "", nil); err != nil { t.Fatal(err) } @@ -149,7 +149,7 @@ func DriverTestCreateSnap(t testing.TB, drivername string, driverOptions ...stri } }() - if err := driver.Create("Snap", "Base", "", nil); err != nil { + if err := driver.Create("Snap", "Base", nil); err != nil { t.Fatal(err) } @@ -169,7 +169,7 @@ func DriverTestDeepLayerRead(t testing.TB, layerCount int, drivername string, dr base := stringid.GenerateRandomID() - if err := driver.Create(base, "", "", nil); err != nil { + if err := driver.Create(base, "", nil); err != nil { t.Fatal(err) } @@ -202,7 +202,7 @@ func DriverTestDiffApply(t testing.TB, fileCount int, drivername string, driverO deleteFile := "file-remove.txt" deleteFileContent := []byte("This file should get removed in upper!") - if err := driver.Create(base, "", "", nil); err != nil { + if err := driver.Create(base, "", nil); err != nil { t.Fatal(err) } @@ -214,7 +214,7 @@ func DriverTestDiffApply(t testing.TB, fileCount int, drivername string, driverO t.Fatal(err) } - if err := driver.Create(upper, base, "", nil); err != nil { + if err := driver.Create(upper, base, nil); err != nil { t.Fatal(err) } @@ -232,7 +232,7 @@ func DriverTestDiffApply(t testing.TB, fileCount int, drivername string, driverO } diff := stringid.GenerateRandomID() - if err := driver.Create(diff, base, "", nil); err != nil { + if err := driver.Create(diff, base, nil); err != nil { t.Fatal(err) } @@ -282,7 +282,7 @@ func DriverTestChanges(t testing.TB, drivername string, driverOptions ...string) base := stringid.GenerateRandomID() upper := stringid.GenerateRandomID() - if err := driver.Create(base, "", "", nil); err != nil { + if err := driver.Create(base, "", nil); err != nil { t.Fatal(err) } @@ -290,7 +290,7 @@ func DriverTestChanges(t testing.TB, drivername string, driverOptions ...string) t.Fatal(err) } - if err := driver.Create(upper, base, "", nil); err != nil { + if err := driver.Create(upper, base, nil); err != nil { t.Fatal(err) } @@ -334,7 +334,7 @@ func DriverTestSetQuota(t *testing.T, drivername string) { createBase(t, driver, "Base") storageOpt := make(map[string]string, 1) storageOpt["size"] = "50M" - if err := driver.Create("zfsTest", "Base", "", storageOpt); err != nil { + if err := driver.Create("zfsTest", "Base", storageOpt); err != nil { t.Fatal(err) } diff --git a/drivers/graphtest/testutil_unix.go b/drivers/graphtest/testutil_unix.go index 732a5b832d..83d974d494 100644 --- a/drivers/graphtest/testutil_unix.go +++ b/drivers/graphtest/testutil_unix.go @@ -94,7 +94,7 @@ func createBase(t testing.TB, driver graphdriver.Driver, name string) { oldmask := syscall.Umask(0) defer syscall.Umask(oldmask) - if err := driver.CreateReadWrite(name, "", "", nil); err != nil { + if err := driver.CreateReadWrite(name, "", nil); err != nil { t.Fatal(err) } diff --git a/drivers/overlay/mount.go b/drivers/overlay/mount.go index 1b53f0c8ae..cf09e12d3c 100644 --- a/drivers/overlay/mount.go +++ b/drivers/overlay/mount.go @@ -9,9 +9,9 @@ import ( "fmt" "os" "runtime" - "syscall" "github.com/containers/storage/pkg/reexec" + "golang.org/x/sys/unix" ) func init() { @@ -31,12 +31,12 @@ type mountOptions struct { Flag uint32 } -func mountFrom(dir, device, target, mType, label string) error { +func mountFrom(dir, device, target, mType string, flags uintptr, label string) error { options := &mountOptions{ Device: device, Target: target, Type: mType, - Flag: 0, + Flag: uint32(flags), Label: label, } @@ -80,7 +80,7 @@ func mountFromMain() { fatal(err) } - if err := syscall.Mount(options.Device, options.Target, options.Type, uintptr(options.Flag), options.Label); err != nil { + if err := unix.Mount(options.Device, options.Target, options.Type, uintptr(options.Flag), options.Label); err != nil { fatal(err) } diff --git a/drivers/overlay/overlay.go b/drivers/overlay/overlay.go index 8c58c59088..1daaff04f9 100644 --- a/drivers/overlay/overlay.go +++ b/drivers/overlay/overlay.go @@ -5,6 +5,7 @@ package overlay import ( "bufio" "fmt" + "io" "io/ioutil" "os" "os/exec" @@ -17,16 +18,23 @@ import ( "github.com/sirupsen/logrus" "github.com/containers/storage/drivers" + "github.com/containers/storage/drivers/overlayutils" + "github.com/containers/storage/drivers/quota" "github.com/containers/storage/pkg/archive" "github.com/containers/storage/pkg/chrootarchive" "github.com/containers/storage/pkg/directory" + "github.com/containers/storage/pkg/fsutils" "github.com/containers/storage/pkg/idtools" + "github.com/containers/storage/pkg/locker" "github.com/containers/storage/pkg/mount" "github.com/containers/storage/pkg/parsers" "github.com/containers/storage/pkg/parsers/kernel" + "github.com/containers/storage/pkg/system" + units "github.com/docker/go-units" "github.com/opencontainers/selinux/go-selinux/label" "github.com/pkg/errors" + "golang.org/x/sys/unix" ) var ( @@ -42,7 +50,7 @@ var ( // Each container/image has at least a "diff" directory and "link" file. // If there is also a "lower" file when there are diff layers -// below as well as "merged" and "work" directories. The "diff" directory +// below as well as "merged" and "work" directories. The "diff" directory // has the upper layer of the overlay and is used to capture any // changes to the layer. The "lower" file contains all the lower layer // mounts separated by ":" and ordered from uppermost to lowermost @@ -76,17 +84,29 @@ const ( idLength = 26 ) +type overlayOptions struct { + overrideKernelCheck bool + imageStores []string + quota quota.Quota +} + // Driver contains information about the home directory and the list of active mounts that are created using this driver. type Driver struct { - name string - home string - uidMaps []idtools.IDMap - gidMaps []idtools.IDMap - ctr *graphdriver.RefCounter - opts *overlayOptions + name string + home string + uidMaps []idtools.IDMap + gidMaps []idtools.IDMap + ctr *graphdriver.RefCounter + quotaCtl *quota.Control + options *overlayOptions + supportsDType bool + locker *locker.Locker } -var backingFs = "" +var ( + backingFs = "" + projectQuotaSupported = false +) func init() { graphdriver.Register("overlay", InitAsOverlay) @@ -114,7 +134,7 @@ func InitWithName(name, home string, options []string, uidMaps, gidMaps []idtool if !opts.overrideKernelCheck { return nil, errors.Wrap(graphdriver.ErrNotSupported, "kernel too old to provide multiple lowers feature for overlay") } - logrus.Warnf("Using pre-4.0.0 kernel for overlay, mount failures may require kernel update") + logrus.Warn("Using pre-4.0.0 kernel for overlay, mount failures may require kernel update") } fsMagic, err := graphdriver.GetFSMagic(home) @@ -127,9 +147,19 @@ func InitWithName(name, home string, options []string, uidMaps, gidMaps []idtool // check if they are running over btrfs, aufs, zfs, overlay, or ecryptfs switch fsMagic { - case graphdriver.FsMagicBtrfs, graphdriver.FsMagicAufs, graphdriver.FsMagicZfs, graphdriver.FsMagicOverlay, graphdriver.FsMagicEcryptfs: + case graphdriver.FsMagicAufs, graphdriver.FsMagicZfs, graphdriver.FsMagicOverlay, graphdriver.FsMagicEcryptfs: logrus.Errorf("'overlay' is not supported over %s", backingFs) return nil, errors.Wrapf(graphdriver.ErrIncompatibleFS, "'overlay' is not supported over %s", backingFs) + case graphdriver.FsMagicBtrfs: + // Support for OverlayFS on BTRFS was added in kernel 4.7 + // See https://btrfs.wiki.kernel.org/index.php/Changelog + if kernel.CompareKernelVersion(*v, kernel.VersionInfo{Kernel: 4, Major: 7, Minor: 0}) < 0 { + if !opts.overrideKernelCheck { + logrus.Errorf("'overlay' requires kernel 4.7 to use on %s", backingFs) + return nil, graphdriver.ErrIncompatibleFS + } + logrus.Warn("Using pre-4.7.0 kernel for overlay on btrfs, may require kernel update") + } } rootUID, rootGID, err := idtools.GetRootUIDGID(uidMaps, gidMaps) @@ -145,15 +175,40 @@ func InitWithName(name, home string, options []string, uidMaps, gidMaps []idtool return nil, err } + supportsDType, err := fsutils.SupportsDType(home) + if err != nil { + return nil, err + } + if !supportsDType { + // not a fatal error until v17.12 (#27443) + logrus.Warn(overlayutils.ErrDTypeNotSupported("overlay", backingFs)) + } + d := &Driver{ - name: name, - home: home, - uidMaps: uidMaps, - gidMaps: gidMaps, - ctr: graphdriver.NewRefCounter(graphdriver.NewFsChecker(graphdriver.FsMagicOverlay)), - opts: opts, + name: name, + home: home, + uidMaps: uidMaps, + gidMaps: gidMaps, + ctr: graphdriver.NewRefCounter(graphdriver.NewFsChecker(graphdriver.FsMagicOverlay)), + supportsDType: supportsDType, + locker: locker.New(), + options: opts, + } + + if backingFs == "xfs" { + // Try to enable project quota support over xfs. + if d.quotaCtl, err = quota.NewControl(home); err == nil { + projectQuotaSupported = true + } else if opts.quota.Size > 0 { + return nil, fmt.Errorf("Storage option overlay.size not supported. Filesystem does not support Project Quota: %v", err) + } + } else if opts.quota.Size > 0 { + // if xfs is not the backing fs then error out if the storage-opt overlay.size is used. + return nil, fmt.Errorf("Storage Option overlay.size only supported for backingFS XFS. Found %v", backingFs) } + logrus.Debugf("backingFs=%s, projectQuotaSupported=%v", backingFs, projectQuotaSupported) + return d, nil } @@ -171,11 +226,6 @@ func InitAsOverlay2(home string, options []string, uidMaps, gidMaps []idtools.ID return InitWithName("overlay2", home, options, uidMaps, gidMaps) } -type overlayOptions struct { - overrideKernelCheck bool - imageStores []string -} - func parseOptions(name string, options []string) (*overlayOptions, error) { o := &overlayOptions{} for _, option := range options { @@ -190,6 +240,12 @@ func parseOptions(name string, options []string) (*overlayOptions, error) { if err != nil { return nil, err } + case "overlay.size": + size, err := units.RAMInBytes(val) + if err != nil { + return nil, err + } + o.quota.Size = uint64(size) case "overlay.imagestore", "overlay2.imagestore": // Additional read only image stores to use for lower paths for _, store := range strings.Split(val, ",") { @@ -243,6 +299,7 @@ func (d *Driver) String() string { func (d *Driver) Status() [][2]string { return [][2]string{ {"Backing Filesystem", backingFs}, + {"Supports d_type", strconv.FormatBool(d.supportsDType)}, } } @@ -280,18 +337,39 @@ func (d *Driver) Cleanup() error { // CreateReadWrite creates a layer that is writable for use as a container // file system. -func (d *Driver) CreateReadWrite(id, parent, mountLabel string, storageOpt map[string]string) error { - return d.Create(id, parent, mountLabel, storageOpt) +func (d *Driver) CreateReadWrite(id, parent string, opts *graphdriver.CreateOpts) error { + if opts != nil && len(opts.StorageOpt) != 0 && !projectQuotaSupported { + return fmt.Errorf("--storage-opt is supported only for overlay over xfs with 'pquota' mount option") + } + + if opts == nil { + opts = &graphdriver.CreateOpts{ + StorageOpt: map[string]string{}, + } + } + + if _, ok := opts.StorageOpt["size"]; !ok { + if opts.StorageOpt == nil { + opts.StorageOpt = map[string]string{} + } + opts.StorageOpt["size"] = strconv.FormatUint(d.options.quota.Size, 10) + } + + return d.create(id, parent, opts) } // Create is used to create the upper, lower, and merge directories required for overlay fs for a given id. // The parent filesystem is used to configure these directories for the overlay. -func (d *Driver) Create(id, parent, mountLabel string, storageOpt map[string]string) (retErr error) { - - if len(storageOpt) != 0 { - return fmt.Errorf("--storage-opt is not supported for overlay") +func (d *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) (retErr error) { + if opts != nil && len(opts.StorageOpt) != 0 { + if _, ok := opts.StorageOpt["size"]; ok { + return fmt.Errorf("--storage-opt size is only supported for ReadWrite Layers") + } } + return d.create(id, parent, opts) +} +func (d *Driver) create(id, parent string, opts *graphdriver.CreateOpts) (retErr error) { dir := d.dir(id) rootUID, rootGID, err := idtools.GetRootUIDGID(d.uidMaps, d.gidMaps) @@ -312,6 +390,20 @@ func (d *Driver) Create(id, parent, mountLabel string, storageOpt map[string]str } }() + if opts != nil && len(opts.StorageOpt) > 0 { + driver := &Driver{} + if err := d.parseStorageOpt(opts.StorageOpt, driver); err != nil { + return err + } + + if driver.options.quota.Size > 0 { + // Set container disk quota limit + if err := d.quotaCtl.SetQuota(dir, driver.options.quota); err != nil { + return err + } + } + } + if err := idtools.MkdirAs(path.Join(dir, "diff"), 0755, rootUID, rootGID); err != nil { return err } @@ -351,6 +443,26 @@ func (d *Driver) Create(id, parent, mountLabel string, storageOpt map[string]str return nil } +// Parse overlay storage options +func (d *Driver) parseStorageOpt(storageOpt map[string]string, driver *Driver) error { + // Read size to set the disk project quota per container + for key, val := range storageOpt { + key := strings.ToLower(key) + switch key { + case "size": + size, err := units.RAMInBytes(val) + if err != nil { + return err + } + driver.options.quota.Size = uint64(size) + default: + return fmt.Errorf("Unknown option %s", key) + } + } + + return nil +} + func (d *Driver) getLower(parent string) (string, error) { parentDir := d.dir(parent) @@ -377,11 +489,11 @@ func (d *Driver) getLower(parent string) (string, error) { return strings.Join(lowers, ":"), nil } -func (d *Driver) dir(val string) string { - newpath := path.Join(d.home, val) +func (d *Driver) dir(id string) string { + newpath := path.Join(d.home, id) if _, err := os.Stat(newpath); err != nil { for _, p := range d.AdditionalImageStores() { - l := path.Join(p, d.name, val) + l := path.Join(p, d.name, id) _, err = os.Stat(l) if err == nil { return l @@ -411,6 +523,8 @@ func (d *Driver) getLowerDirs(id string) ([]string, error) { // Remove cleans the directories that are created for this id. func (d *Driver) Remove(id string) error { + d.locker.Lock(id) + defer d.locker.Unlock(id) dir := d.dir(id) lid, err := ioutil.ReadFile(path.Join(dir, "link")) if err == nil { @@ -419,7 +533,7 @@ func (d *Driver) Remove(id string) error { } } - if err := os.RemoveAll(dir); err != nil && !os.IsNotExist(err) { + if err := system.EnsureRemoveAll(dir); err != nil && !os.IsNotExist(err) { return err } return nil @@ -427,6 +541,8 @@ func (d *Driver) Remove(id string) error { // Get creates and mounts the required file system for the given id and returns the mount path. func (d *Driver) Get(id string, mountLabel string) (s string, err error) { + d.locker.Lock(id) + defer d.locker.Unlock(id) dir := d.dir(id) if _, err := os.Stat(dir); err != nil { return "", err @@ -474,7 +590,7 @@ func (d *Driver) Get(id string, mountLabel string) (s string, err error) { defer func() { if err != nil { if c := d.ctr.Decrement(mergedDir); c <= 0 { - syscall.Unmount(mergedDir, 0) + unix.Unmount(mergedDir, 0) } } }() @@ -486,7 +602,7 @@ func (d *Driver) Get(id string, mountLabel string) (s string, err error) { return "", fmt.Errorf("cannot mount layer, mount label too large %d", len(mountLabel)) } - if err := mountFrom(d.home, "overlay", path.Join(id, "merged"), "overlay", mountLabel); err != nil { + if err := mountFrom(d.home, "overlay", path.Join(id, "merged"), "overlay", 0, mountLabel); err != nil { return "", fmt.Errorf("error creating overlay mount to %s: %v", mergedDir, err) } @@ -506,11 +622,13 @@ func (d *Driver) Get(id string, mountLabel string) (s string, err error) { // Put unmounts the mount path created for the give id. func (d *Driver) Put(id string) error { + d.locker.Lock(id) + defer d.locker.Unlock(id) mountpoint := path.Join(d.dir(id), "merged") if count := d.ctr.Decrement(mountpoint); count > 0 { return nil } - err := syscall.Unmount(mountpoint, 0) + err := unix.Unmount(mountpoint, 0) if err != nil { if _, err := ioutil.ReadFile(path.Join(d.dir(id), lowerFile)); err != nil { // We didn't have a "lower" directory, so we weren't mounting a "merged" directory anyway @@ -527,8 +645,30 @@ func (d *Driver) Exists(id string) bool { return err == nil } +// isParent returns if the passed in parent is the direct parent of the passed in layer +func (d *Driver) isParent(id, parent string) bool { + lowers, err := d.getLowerDirs(id) + if err != nil { + return false + } + if parent == "" && len(lowers) > 0 { + return false + } + + parentDir := d.dir(parent) + var ld string + if len(lowers) > 0 { + ld = filepath.Dir(lowers[0]) + } + if ld == "" && parent == "" { + return true + } + return ld == parentDir +} + // ApplyDiff applies the new layer into a root -func (d *Driver) ApplyDiff(id string, parent string, diff archive.Reader) (size int64, err error) { +func (d *Driver) ApplyDiff(id string, parent string, diff io.Reader) (size int64, err error) { + applyDir := d.getDiffPath(id) logrus.Debugf("Applying tar in %s", applyDir) @@ -541,7 +681,7 @@ func (d *Driver) ApplyDiff(id string, parent string, diff archive.Reader) (size return 0, err } - return d.DiffSize(id, parent) + return directory.Size(applyDir) } func (d *Driver) getDiffPath(id string) string { @@ -559,7 +699,8 @@ func (d *Driver) DiffSize(id, parent string) (size int64, err error) { // Diff produces an archive of the changes between the specified // layer and its parent layer which may be "". -func (d *Driver) Diff(id, parent string) (archive.Archive, error) { +func (d *Driver) Diff(id, parent string) (io.ReadCloser, error) { + diffPath := d.getDiffPath(id) logrus.Debugf("Tar with options on %s", diffPath) return archive.TarWithOptions(diffPath, &archive.TarOptions{ @@ -573,6 +714,7 @@ func (d *Driver) Diff(id, parent string) (archive.Archive, error) { // Changes produces a list of changes between the specified layer // and its parent layer. If parent is "", then all changes will be ADD changes. func (d *Driver) Changes(id, parent string) ([]archive.Change, error) { + // Overlay doesn't have snapshots, so we need to get changes from all parent // layers. diffPath := d.getDiffPath(id) @@ -586,5 +728,5 @@ func (d *Driver) Changes(id, parent string) ([]archive.Change, error) { // AdditionalImageStores returns additional image stores supported by the driver func (d *Driver) AdditionalImageStores() []string { - return d.opts.imageStores + return d.options.imageStores } diff --git a/drivers/overlay/overlay_test.go b/drivers/overlay/overlay_test.go index 39fd09fde9..d5200fd247 100644 --- a/drivers/overlay/overlay_test.go +++ b/drivers/overlay/overlay_test.go @@ -3,18 +3,17 @@ package overlay import ( + "io/ioutil" "os" - "syscall" "testing" "github.com/containers/storage/drivers" "github.com/containers/storage/drivers/graphtest" "github.com/containers/storage/pkg/archive" "github.com/containers/storage/pkg/reexec" + "golang.org/x/sys/unix" ) -const driverName = "overlay" - func init() { // Do not sure chroot to speed run time and allow archive // errors or hangs to be debugged directly from the test process. @@ -32,7 +31,19 @@ func cdMountFrom(dir, device, target, mType, label string) error { os.Chdir(dir) defer os.Chdir(wd) - return syscall.Mount(device, target, mType, 0, label) + return unix.Mount(device, target, mType, 0, label) +} + +func skipIfNaive(t *testing.T) { + td, err := ioutil.TempDir("", "naive-check-") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + defer os.RemoveAll(td) + + if useNaiveDiff(td) { + t.Skipf("Cannot run test with naive diff") + } } // This avoids creating a new driver for each test if all tests are run @@ -58,10 +69,12 @@ func TestOverlay128LayerRead(t *testing.T) { } func TestOverlayDiffApply10Files(t *testing.T) { + skipIfNaive(t) graphtest.DriverTestDiffApply(t, 10, driverName) } func TestOverlayChanges(t *testing.T) { + skipIfNaive(t) graphtest.DriverTestChanges(t, driverName) } diff --git a/drivers/overlay/randomid.go b/drivers/overlay/randomid.go index 975b3a50fb..fc565ef0ba 100644 --- a/drivers/overlay/randomid.go +++ b/drivers/overlay/randomid.go @@ -12,6 +12,7 @@ import ( "time" "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" ) // generateID creates a new random string identifier with the given length @@ -69,7 +70,7 @@ func retryOnError(err error) bool { case *os.PathError: return retryOnError(err.Err) // unpack the target error case syscall.Errno: - if err == syscall.EPERM { + if err == unix.EPERM { // EPERM represents an entropy pool exhaustion, a condition under // which we backoff and retry. return true diff --git a/drivers/overlayutils/overlayutils.go b/drivers/overlayutils/overlayutils.go new file mode 100644 index 0000000000..7491c3457c --- /dev/null +++ b/drivers/overlayutils/overlayutils.go @@ -0,0 +1,18 @@ +// +build linux + +package overlayutils + +import ( + "errors" + "fmt" +) + +// ErrDTypeNotSupported denotes that the backing filesystem doesn't support d_type. +func ErrDTypeNotSupported(driver, backingFs string) error { + msg := fmt.Sprintf("%s: the backing %s filesystem is formatted without d_type support, which leads to incorrect behavior.", driver, backingFs) + if backingFs == "xfs" { + msg += " Reformat the filesystem with ftype=1 to enable d_type support." + } + msg += " Running without d_type support will no longer be supported in Docker 17.12." + return errors.New(msg) +} diff --git a/drivers/proxy.go b/drivers/proxy.go index d56b8731f1..0159972a08 100644 --- a/drivers/proxy.go +++ b/drivers/proxy.go @@ -54,11 +54,11 @@ func (d *graphDriverProxy) String() string { return d.name } -func (d *graphDriverProxy) CreateReadWrite(id, parent, mountLabel string, storageOpt map[string]string) error { +func (d *graphDriverProxy) CreateReadWrite(id, parent, opts *CreateOpts) error { args := &graphDriverRequest{ ID: id, Parent: parent, - MountLabel: mountLabel, + MountLabel: opts.MountLabel, } var ret graphDriverResponse if err := d.client.Call("GraphDriver.CreateReadWrite", args, &ret); err != nil { @@ -70,11 +70,11 @@ func (d *graphDriverProxy) CreateReadWrite(id, parent, mountLabel string, storag return nil } -func (d *graphDriverProxy) Create(id, parent, mountLabel string, storageOpt map[string]string) error { +func (d *graphDriverProxy) Create(id, parent, opts *CreateOpts) error { args := &graphDriverRequest{ ID: id, Parent: parent, - MountLabel: mountLabel, + MountLabel: opts.MountLabel, } var ret graphDriverResponse if err := d.client.Call("GraphDriver.Create", args, &ret); err != nil { diff --git a/drivers/quota/projectquota.go b/drivers/quota/projectquota.go new file mode 100644 index 0000000000..0e70515434 --- /dev/null +++ b/drivers/quota/projectquota.go @@ -0,0 +1,337 @@ +// +build linux + +// +// projectquota.go - implements XFS project quota controls +// for setting quota limits on a newly created directory. +// It currently supports the legacy XFS specific ioctls. +// +// TODO: use generic quota control ioctl FS_IOC_FS{GET,SET}XATTR +// for both xfs/ext4 for kernel version >= v4.5 +// + +package quota + +/* +#include +#include +#include +#include +#include + +#ifndef FS_XFLAG_PROJINHERIT +struct fsxattr { + __u32 fsx_xflags; + __u32 fsx_extsize; + __u32 fsx_nextents; + __u32 fsx_projid; + unsigned char fsx_pad[12]; +}; +#define FS_XFLAG_PROJINHERIT 0x00000200 +#endif +#ifndef FS_IOC_FSGETXATTR +#define FS_IOC_FSGETXATTR _IOR ('X', 31, struct fsxattr) +#endif +#ifndef FS_IOC_FSSETXATTR +#define FS_IOC_FSSETXATTR _IOW ('X', 32, struct fsxattr) +#endif + +#ifndef PRJQUOTA +#define PRJQUOTA 2 +#endif +#ifndef XFS_PROJ_QUOTA +#define XFS_PROJ_QUOTA 2 +#endif +#ifndef Q_XSETPQLIM +#define Q_XSETPQLIM QCMD(Q_XSETQLIM, PRJQUOTA) +#endif +#ifndef Q_XGETPQUOTA +#define Q_XGETPQUOTA QCMD(Q_XGETQUOTA, PRJQUOTA) +#endif +*/ +import "C" +import ( + "fmt" + "io/ioutil" + "path" + "path/filepath" + "unsafe" + + "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" +) + +// Quota limit params - currently we only control blocks hard limit +type Quota struct { + Size uint64 +} + +// Control - Context to be used by storage driver (e.g. overlay) +// who wants to apply project quotas to container dirs +type Control struct { + backingFsBlockDev string + nextProjectID uint32 + quotas map[string]uint32 +} + +// NewControl - initialize project quota support. +// Test to make sure that quota can be set on a test dir and find +// the first project id to be used for the next container create. +// +// Returns nil (and error) if project quota is not supported. +// +// First get the project id of the home directory. +// This test will fail if the backing fs is not xfs. +// +// xfs_quota tool can be used to assign a project id to the driver home directory, e.g.: +// echo 999:/var/lib/docker/overlay2 >> /etc/projects +// echo docker:999 >> /etc/projid +// xfs_quota -x -c 'project -s docker' / +// +// In that case, the home directory project id will be used as a "start offset" +// and all containers will be assigned larger project ids (e.g. >= 1000). +// This is a way to prevent xfs_quota management from conflicting with docker. +// +// Then try to create a test directory with the next project id and set a quota +// on it. If that works, continue to scan existing containers to map allocated +// project ids. +// +func NewControl(basePath string) (*Control, error) { + // + // Get project id of parent dir as minimal id to be used by driver + // + minProjectID, err := getProjectID(basePath) + if err != nil { + return nil, err + } + minProjectID++ + + // + // create backing filesystem device node + // + backingFsBlockDev, err := makeBackingFsDev(basePath) + if err != nil { + return nil, err + } + + // + // Test if filesystem supports project quotas by trying to set + // a quota on the first available project id + // + quota := Quota{ + Size: 0, + } + if err := setProjectQuota(backingFsBlockDev, minProjectID, quota); err != nil { + return nil, err + } + + q := Control{ + backingFsBlockDev: backingFsBlockDev, + nextProjectID: minProjectID + 1, + quotas: make(map[string]uint32), + } + + // + // get first project id to be used for next container + // + err = q.findNextProjectID(basePath) + if err != nil { + return nil, err + } + + logrus.Debugf("NewControl(%s): nextProjectID = %d", basePath, q.nextProjectID) + return &q, nil +} + +// SetQuota - assign a unique project id to directory and set the quota limits +// for that project id +func (q *Control) SetQuota(targetPath string, quota Quota) error { + + projectID, ok := q.quotas[targetPath] + if !ok { + projectID = q.nextProjectID + + // + // assign project id to new container directory + // + err := setProjectID(targetPath, projectID) + if err != nil { + return err + } + + q.quotas[targetPath] = projectID + q.nextProjectID++ + } + + // + // set the quota limit for the container's project id + // + logrus.Debugf("SetQuota(%s, %d): projectID=%d", targetPath, quota.Size, projectID) + return setProjectQuota(q.backingFsBlockDev, projectID, quota) +} + +// setProjectQuota - set the quota for project id on xfs block device +func setProjectQuota(backingFsBlockDev string, projectID uint32, quota Quota) error { + var d C.fs_disk_quota_t + d.d_version = C.FS_DQUOT_VERSION + d.d_id = C.__u32(projectID) + d.d_flags = C.XFS_PROJ_QUOTA + + d.d_fieldmask = C.FS_DQ_BHARD | C.FS_DQ_BSOFT + d.d_blk_hardlimit = C.__u64(quota.Size / 512) + d.d_blk_softlimit = d.d_blk_hardlimit + + var cs = C.CString(backingFsBlockDev) + defer C.free(unsafe.Pointer(cs)) + + _, _, errno := unix.Syscall6(unix.SYS_QUOTACTL, C.Q_XSETPQLIM, + uintptr(unsafe.Pointer(cs)), uintptr(d.d_id), + uintptr(unsafe.Pointer(&d)), 0, 0) + if errno != 0 { + return fmt.Errorf("Failed to set quota limit for projid %d on %s: %v", + projectID, backingFsBlockDev, errno.Error()) + } + + return nil +} + +// GetQuota - get the quota limits of a directory that was configured with SetQuota +func (q *Control) GetQuota(targetPath string, quota *Quota) error { + + projectID, ok := q.quotas[targetPath] + if !ok { + return fmt.Errorf("quota not found for path : %s", targetPath) + } + + // + // get the quota limit for the container's project id + // + var d C.fs_disk_quota_t + + var cs = C.CString(q.backingFsBlockDev) + defer C.free(unsafe.Pointer(cs)) + + _, _, errno := unix.Syscall6(unix.SYS_QUOTACTL, C.Q_XGETPQUOTA, + uintptr(unsafe.Pointer(cs)), uintptr(C.__u32(projectID)), + uintptr(unsafe.Pointer(&d)), 0, 0) + if errno != 0 { + return fmt.Errorf("Failed to get quota limit for projid %d on %s: %v", + projectID, q.backingFsBlockDev, errno.Error()) + } + quota.Size = uint64(d.d_blk_hardlimit) * 512 + + return nil +} + +// getProjectID - get the project id of path on xfs +func getProjectID(targetPath string) (uint32, error) { + dir, err := openDir(targetPath) + if err != nil { + return 0, err + } + defer closeDir(dir) + + var fsx C.struct_fsxattr + _, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.FS_IOC_FSGETXATTR, + uintptr(unsafe.Pointer(&fsx))) + if errno != 0 { + return 0, fmt.Errorf("Failed to get projid for %s: %v", targetPath, errno.Error()) + } + + return uint32(fsx.fsx_projid), nil +} + +// setProjectID - set the project id of path on xfs +func setProjectID(targetPath string, projectID uint32) error { + dir, err := openDir(targetPath) + if err != nil { + return err + } + defer closeDir(dir) + + var fsx C.struct_fsxattr + _, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.FS_IOC_FSGETXATTR, + uintptr(unsafe.Pointer(&fsx))) + if errno != 0 { + return fmt.Errorf("Failed to get projid for %s: %v", targetPath, errno.Error()) + } + fsx.fsx_projid = C.__u32(projectID) + fsx.fsx_xflags |= C.FS_XFLAG_PROJINHERIT + _, _, errno = unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.FS_IOC_FSSETXATTR, + uintptr(unsafe.Pointer(&fsx))) + if errno != 0 { + return fmt.Errorf("Failed to set projid for %s: %v", targetPath, errno.Error()) + } + + return nil +} + +// findNextProjectID - find the next project id to be used for containers +// by scanning driver home directory to find used project ids +func (q *Control) findNextProjectID(home string) error { + files, err := ioutil.ReadDir(home) + if err != nil { + return fmt.Errorf("read directory failed : %s", home) + } + for _, file := range files { + if !file.IsDir() { + continue + } + path := filepath.Join(home, file.Name()) + projid, err := getProjectID(path) + if err != nil { + return err + } + if projid > 0 { + q.quotas[path] = projid + } + if q.nextProjectID <= projid { + q.nextProjectID = projid + 1 + } + } + + return nil +} + +func free(p *C.char) { + C.free(unsafe.Pointer(p)) +} + +func openDir(path string) (*C.DIR, error) { + Cpath := C.CString(path) + defer free(Cpath) + + dir := C.opendir(Cpath) + if dir == nil { + return nil, fmt.Errorf("Can't open dir") + } + return dir, nil +} + +func closeDir(dir *C.DIR) { + if dir != nil { + C.closedir(dir) + } +} + +func getDirFd(dir *C.DIR) uintptr { + return uintptr(C.dirfd(dir)) +} + +// Get the backing block device of the driver home directory +// and create a block device node under the home directory +// to be used by quotactl commands +func makeBackingFsDev(home string) (string, error) { + var stat unix.Stat_t + if err := unix.Stat(home, &stat); err != nil { + return "", err + } + + backingFsBlockDev := path.Join(home, "backingFsBlockDev") + // Re-create just in case someone copied the home directory over to a new device + unix.Unlink(backingFsBlockDev) + if err := unix.Mknod(backingFsBlockDev, unix.S_IFBLK|0600, int(stat.Dev)); err != nil { + return "", fmt.Errorf("Failed to mknod %s: %v", backingFsBlockDev, err) + } + + return backingFsBlockDev, nil +} diff --git a/drivers/vfs/driver.go b/drivers/vfs/driver.go index ff7a88f1a2..129b1a73e0 100644 --- a/drivers/vfs/driver.go +++ b/drivers/vfs/driver.go @@ -8,13 +8,13 @@ import ( "github.com/containers/storage/drivers" "github.com/containers/storage/pkg/chrootarchive" "github.com/containers/storage/pkg/idtools" - + "github.com/containers/storage/pkg/system" "github.com/opencontainers/selinux/go-selinux/label" ) var ( // CopyWithTar defines the copy method to use. - CopyWithTar = chrootarchive.CopyWithTar + CopyWithTar = chrootarchive.NewArchiver(nil).CopyWithTar ) func init() { @@ -25,15 +25,11 @@ func init() { // This sets the home directory for the driver and returns NaiveDiffDriver. func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (graphdriver.Driver, error) { d := &Driver{ - home: home, - uidMaps: uidMaps, - gidMaps: gidMaps, + home: home, + idMappings: idtools.NewIDMappingsFromMaps(uidMaps, gidMaps), } - rootUID, rootGID, err := idtools.GetRootUIDGID(uidMaps, gidMaps) - if err != nil { - return nil, err - } - if err := idtools.MkdirAllAs(home, 0700, rootUID, rootGID); err != nil { + rootIDs := d.idMappings.RootPair() + if err := idtools.MkdirAllAndChown(home, 0700, rootIDs); err != nil { return nil, err } return graphdriver.NewNaiveDiffDriver(d, uidMaps, gidMaps), nil @@ -44,9 +40,8 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap // In order to support layering, files are copied from the parent layer into the new layer. There is no copy-on-write support. // Driver must be wrapped in NaiveDiffDriver to be used as a graphdriver.Driver type Driver struct { - home string - uidMaps []idtools.IDMap - gidMaps []idtools.IDMap + home string + idMappings *idtools.IDMappings } func (d *Driver) String() string { @@ -70,29 +65,26 @@ func (d *Driver) Cleanup() error { // CreateReadWrite creates a layer that is writable for use as a container // file system. -func (d *Driver) CreateReadWrite(id, parent, mountLabel string, storageOpt map[string]string) error { - return d.Create(id, parent, mountLabel, storageOpt) +func (d *Driver) CreateReadWrite(id, parent string, opts *graphdriver.CreateOpts) error { + return d.Create(id, parent, opts) } // Create prepares the filesystem for the VFS driver and copies the directory for the given id under the parent. -func (d *Driver) Create(id, parent, mountLabel string, storageOpt map[string]string) error { - if len(storageOpt) != 0 { +func (d *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) error { + if opts != nil && len(opts.StorageOpt) != 0 { return fmt.Errorf("--storage-opt is not supported for vfs") } dir := d.dir(id) - rootUID, rootGID, err := idtools.GetRootUIDGID(d.uidMaps, d.gidMaps) - if err != nil { + rootIDs := d.idMappings.RootPair() + if err := idtools.MkdirAllAndChown(filepath.Dir(dir), 0700, rootIDs); err != nil { return err } - if err := idtools.MkdirAllAs(filepath.Dir(dir), 0700, rootUID, rootGID); err != nil { + if err := idtools.MkdirAndChown(dir, 0755, rootIDs); err != nil { return err } - if err := idtools.MkdirAs(dir, 0755, rootUID, rootGID); err != nil { - return err - } - opts := []string{"level:s0"} - if _, mountLabel, err := label.InitLabels(opts); err == nil { + labelOpts := []string{"level:s0"} + if _, mountLabel, err := label.InitLabels(labelOpts); err == nil { label.SetFileLabel(dir, mountLabel) } if parent == "" { @@ -102,10 +94,7 @@ func (d *Driver) Create(id, parent, mountLabel string, storageOpt map[string]str if err != nil { return fmt.Errorf("%s: %s", parent, err) } - if err := CopyWithTar(parentDir, dir); err != nil { - return err - } - return nil + return CopyWithTar(parentDir, dir) } func (d *Driver) dir(id string) string { @@ -114,7 +103,7 @@ func (d *Driver) dir(id string) string { // Remove deletes the content from the directory for a given id. func (d *Driver) Remove(id string) error { - if err := os.RemoveAll(d.dir(id)); err != nil && !os.IsNotExist(err) { + if err := system.EnsureRemoveAll(d.dir(id)); err != nil { return err } return nil diff --git a/drivers/vfs/vfs_test.go b/drivers/vfs/vfs_test.go index f61edafe1b..41a95a084b 100644 --- a/drivers/vfs/vfs_test.go +++ b/drivers/vfs/vfs_test.go @@ -5,7 +5,7 @@ package vfs import ( "testing" - "github.com/containers/storage/drivers/graphtest" + "github.com/containers/storage/daemon/graphdriver/graphtest" "github.com/containers/storage/pkg/reexec" ) diff --git a/drivers/windows/windows.go b/drivers/windows/windows.go index 7ab36513c1..83e9fc8ab9 100644 --- a/drivers/windows/windows.go +++ b/drivers/windows/windows.go @@ -106,13 +106,13 @@ func (d *Driver) Exists(id string) bool { // CreateReadWrite creates a layer that is writable for use as a container // file system. -func (d *Driver) CreateReadWrite(id, parent, mountLabel string, storageOpt map[string]string) error { - return d.create(id, parent, mountLabel, false, storageOpt) +func (d *Driver) CreateReadWrite(id, parent string, opts *CreateOpts) error { + return d.create(id, parent, false, opts) } // Create creates a new read-only layer with the given id. -func (d *Driver) Create(id, parent, mountLabel string, storageOpt map[string]string) error { - return d.create(id, parent, mountLabel, true, storageOpt) +func (d *Driver) Create(id, parent string, opts *CreateOpts) error { + return d.create(id, parent, opts.MountLabel, true, opts.StorageOpt) } func (d *Driver) create(id, parent, mountLabel string, readOnly bool, storageOpt map[string]string) error { diff --git a/drivers/zfs/zfs.go b/drivers/zfs/zfs.go index c9860ec28a..e7db10ea4d 100644 --- a/drivers/zfs/zfs.go +++ b/drivers/zfs/zfs.go @@ -10,7 +10,6 @@ import ( "strconv" "strings" "sync" - "syscall" "time" "github.com/containers/storage/drivers" @@ -19,8 +18,8 @@ import ( "github.com/containers/storage/pkg/parsers" zfs "github.com/mistifyio/go-zfs" "github.com/opencontainers/selinux/go-selinux/label" - "github.com/pkg/errors" "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" ) type zfsOptions struct { @@ -48,13 +47,13 @@ func Init(base string, opt []string, uidMaps, gidMaps []idtools.IDMap) (graphdri if _, err := exec.LookPath("zfs"); err != nil { logrus.Debugf("[zfs] zfs command is not available: %v", err) - return nil, errors.Wrap(graphdriver.ErrPrerequisites, "the 'zfs' command is not available") + return nil, graphdriver.ErrPrerequisites } file, err := os.OpenFile("/dev/zfs", os.O_RDWR, 600) if err != nil { logrus.Debugf("[zfs] cannot open /dev/zfs: %v", err) - return nil, errors.Wrapf(graphdriver.ErrPrerequisites, "could not open /dev/zfs: %v", err) + return nil, graphdriver.ErrPrerequisites } defer file.Close() @@ -100,6 +99,14 @@ func Init(base string, opt []string, uidMaps, gidMaps []idtools.IDMap) (graphdri return nil, fmt.Errorf("BUG: zfs get all -t filesystem -rHp '%s' should contain '%s'", options.fsName, options.fsName) } + rootUID, rootGID, err := idtools.GetRootUIDGID(uidMaps, gidMaps) + if err != nil { + return nil, fmt.Errorf("Failed to get root uid/guid: %v", err) + } + if err := idtools.MkdirAllAs(base, 0700, rootUID, rootGID); err != nil { + return nil, fmt.Errorf("Failed to create '%s': %v", base, err) + } + if err := mount.MakePrivate(base); err != nil { return nil, err } @@ -134,8 +141,8 @@ func parseOptions(opt []string) (zfsOptions, error) { } func lookupZfsDataset(rootdir string) (string, error) { - var stat syscall.Stat_t - if err := syscall.Stat(rootdir, &stat); err != nil { + var stat unix.Stat_t + if err := unix.Stat(rootdir, &stat); err != nil { return "", fmt.Errorf("Failed to access '%s': %s", rootdir, err) } wantedDev := stat.Dev @@ -145,7 +152,7 @@ func lookupZfsDataset(rootdir string) (string, error) { return "", err } for _, m := range mounts { - if err := syscall.Stat(m.Mountpoint, &stat); err != nil { + if err := unix.Stat(m.Mountpoint, &stat); err != nil { logrus.Debugf("[zfs] failed to stat '%s' while scanning for zfs mount: %v", m.Mountpoint, err) continue // may fail on fuse file systems } @@ -213,7 +220,10 @@ func (d *Driver) Status() [][2]string { // Metadata returns image/container metadata related to graph driver func (d *Driver) Metadata(id string) (map[string]string, error) { - return nil, nil + return map[string]string{ + "Mountpoint": d.mountPath(id), + "Dataset": d.zfsPath(id), + }, nil } func (d *Driver) cloneFilesystem(name, parentName string) error { @@ -248,12 +258,17 @@ func (d *Driver) mountPath(id string) string { // CreateReadWrite creates a layer that is writable for use as a container // file system. -func (d *Driver) CreateReadWrite(id, parent, mountLabel string, storageOpt map[string]string) error { - return d.Create(id, parent, mountLabel, storageOpt) +func (d *Driver) CreateReadWrite(id, parent string, opts *graphdriver.CreateOpts) error { + return d.Create(id, parent, opts) } // Create prepares the dataset and filesystem for the ZFS driver for the given id under the parent. -func (d *Driver) Create(id string, parent string, mountLabel string, storageOpt map[string]string) error { +func (d *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) error { + var storageOpt map[string]string + if opts != nil { + storageOpt = opts.StorageOpt + } + err := d.create(id, parent, storageOpt) if err == nil { return nil @@ -391,11 +406,10 @@ func (d *Driver) Put(id string) error { logrus.Debugf(`[zfs] unmount("%s")`, mountpoint) - err = mount.Unmount(mountpoint) - if err != nil { + if err := mount.Unmount(mountpoint); err != nil { return fmt.Errorf("error unmounting to %s: %v", mountpoint, err) } - return err + return nil } // Exists checks to see if the cache entry exists for the given id. diff --git a/drivers/zfs/zfs_freebsd.go b/drivers/zfs/zfs_freebsd.go index ade71b15d7..4ece6ed295 100644 --- a/drivers/zfs/zfs_freebsd.go +++ b/drivers/zfs/zfs_freebsd.go @@ -3,23 +3,22 @@ package zfs import ( "fmt" "strings" - "syscall" - "github.com/containers/storage/drivers" - "github.com/pkg/errors" + "github.com/containers/storage/daemon/graphdriver" "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" ) func checkRootdirFs(rootdir string) error { - var buf syscall.Statfs_t - if err := syscall.Statfs(rootdir, &buf); err != nil { + var buf unix.Statfs_t + if err := unix.Statfs(rootdir, &buf); err != nil { return fmt.Errorf("Failed to access '%s': %s", rootdir, err) } // on FreeBSD buf.Fstypename contains ['z', 'f', 's', 0 ... ] if (buf.Fstypename[0] != 122) || (buf.Fstypename[1] != 102) || (buf.Fstypename[2] != 115) || (buf.Fstypename[3] != 0) { logrus.Debugf("[zfs] no zfs dataset found for rootdir '%s'", rootdir) - return errors.Wrapf(graphdriver.ErrPrerequisites, "no zfs dataset found for rootdir '%s'", rootdir) + return graphdriver.ErrPrerequisites } return nil diff --git a/drivers/zfs/zfs_linux.go b/drivers/zfs/zfs_linux.go index 92b3875690..6c5e9a9fea 100644 --- a/drivers/zfs/zfs_linux.go +++ b/drivers/zfs/zfs_linux.go @@ -2,22 +2,21 @@ package zfs import ( "fmt" - "syscall" "github.com/containers/storage/drivers" - "github.com/pkg/errors" "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" ) func checkRootdirFs(rootdir string) error { - var buf syscall.Statfs_t - if err := syscall.Statfs(rootdir, &buf); err != nil { + var buf unix.Statfs_t + if err := unix.Statfs(rootdir, &buf); err != nil { return fmt.Errorf("Failed to access '%s': %s", rootdir, err) } if graphdriver.FsMagic(buf.Type) != graphdriver.FsMagicZfs { logrus.Debugf("[zfs] no zfs dataset found for rootdir '%s'", rootdir) - return errors.Wrapf(graphdriver.ErrPrerequisites, "no zfs dataset found for rootdir '%s'", rootdir) + return graphdriver.ErrPrerequisites } return nil diff --git a/drivers/zfs/zfs_solaris.go b/drivers/zfs/zfs_solaris.go index ca59563810..41041d06d7 100644 --- a/drivers/zfs/zfs_solaris.go +++ b/drivers/zfs/zfs_solaris.go @@ -20,9 +20,8 @@ import ( "strings" "unsafe" - "github.com/containers/storage/drivers" - "github.com/pkg/errors" - log "github.com/sirupsen/logrus" + "github.com/containers/storage/daemon/graphdriver" + "github.com/sirupsen/logrus" ) func checkRootdirFs(rootdir string) error { @@ -33,9 +32,9 @@ func checkRootdirFs(rootdir string) error { // on Solaris buf.f_basetype contains ['z', 'f', 's', 0 ... ] if (buf.f_basetype[0] != 122) || (buf.f_basetype[1] != 102) || (buf.f_basetype[2] != 115) || (buf.f_basetype[3] != 0) { - log.Debugf("[zfs] no zfs dataset found for rootdir '%s'", rootdir) + logrus.Debugf("[zfs] no zfs dataset found for rootdir '%s'", rootdir) C.free(unsafe.Pointer(buf)) - return errors.Wrapf(graphdriver.ErrPrerequisites, "no zfs dataset found for rootdir '%s'", rootdir) + return graphdriver.ErrPrerequisites } C.free(unsafe.Pointer(buf)) diff --git a/drivers/zfs/zfs_test.go b/drivers/zfs/zfs_test.go index 461bd3e46b..51d71157fc 100644 --- a/drivers/zfs/zfs_test.go +++ b/drivers/zfs/zfs_test.go @@ -5,7 +5,7 @@ package zfs import ( "testing" - "github.com/containers/storage/drivers/graphtest" + "github.com/containers/storage/daemon/graphdriver/graphtest" ) // This avoids creating a new driver for each test if all tests are run diff --git a/layers.go b/layers.go index 774aa65495..aeb140931c 100644 --- a/layers.go +++ b/layers.go @@ -184,7 +184,7 @@ type LayerStore interface { CreateWithFlags(id, parent string, names []string, mountLabel string, options map[string]string, writeable bool, flags map[string]interface{}) (layer *Layer, err error) // Put combines the functions of CreateWithFlags and ApplyDiff. - Put(id, parent string, names []string, mountLabel string, options map[string]string, writeable bool, flags map[string]interface{}, diff archive.Reader) (*Layer, int64, error) + Put(id, parent string, names []string, mountLabel string, options map[string]string, writeable bool, flags map[string]interface{}, diff io.Reader) (*Layer, int64, error) // SetNames replaces the list of names associated with a layer with the // supplied values. @@ -206,7 +206,7 @@ type LayerStore interface { // ApplyDiff reads a tarstream which was created by a previous call to Diff and // applies its changes to a specified layer. - ApplyDiff(to string, diff archive.Reader) (int64, error) + ApplyDiff(to string, diff io.Reader) (int64, error) } type layerStore struct { @@ -463,7 +463,7 @@ func (r *layerStore) Status() ([][2]string, error) { return r.driver.Status(), nil } -func (r *layerStore) Put(id, parent string, names []string, mountLabel string, options map[string]string, writeable bool, flags map[string]interface{}, diff archive.Reader) (layer *Layer, size int64, err error) { +func (r *layerStore) Put(id, parent string, names []string, mountLabel string, options map[string]string, writeable bool, flags map[string]interface{}, diff io.Reader) (layer *Layer, size int64, err error) { if !r.IsReadWrite() { return nil, -1, errors.Wrapf(ErrStoreIsReadOnly, "not allowed to create new layers at %q", r.layerspath()) } @@ -495,10 +495,14 @@ func (r *layerStore) Put(id, parent string, names []string, mountLabel string, o return nil, -1, ErrDuplicateName } } + opts := drivers.CreateOpts{ + MountLabel: mountLabel, + StorageOpt: options, + } if writeable { - err = r.driver.CreateReadWrite(id, parent, mountLabel, options) + err = r.driver.CreateReadWrite(id, parent, &opts) } else { - err = r.driver.Create(id, parent, mountLabel, options) + err = r.driver.Create(id, parent, &opts) } if err == nil { layer = &Layer{ @@ -900,7 +904,7 @@ func (r *layerStore) DiffSize(from, to string) (size int64, err error) { return r.driver.DiffSize(to, from) } -func (r *layerStore) ApplyDiff(to string, diff archive.Reader) (size int64, err error) { +func (r *layerStore) ApplyDiff(to string, diff io.Reader) (size int64, err error) { if !r.IsReadWrite() { return -1, errors.Wrapf(ErrStoreIsReadOnly, "not allowed to modify layer contents at %q", r.layerspath()) } diff --git a/store.go b/store.go index 235a3ad2dc..f28c6395e8 100644 --- a/store.go +++ b/store.go @@ -171,7 +171,7 @@ type Store interface { // if reexec.Init { // return // } - PutLayer(id, parent string, names []string, mountLabel string, writeable bool, diff archive.Reader) (*Layer, int64, error) + PutLayer(id, parent string, names []string, mountLabel string, writeable bool, diff io.Reader) (*Layer, int64, error) // CreateImage creates a new image, optionally with the specified ID // (one will be assigned if none is specified), with optional names, @@ -286,7 +286,7 @@ type Store interface { // if reexec.Init { // return // } - ApplyDiff(to string, diff archive.Reader) (int64, error) + ApplyDiff(to string, diff io.Reader) (int64, error) // LayersByCompressedDigest returns a slice of the layers with the // specified compressed digest value recorded for them. @@ -722,7 +722,7 @@ func (s *store) ContainerStore() (ContainerStore, error) { return nil, ErrLoadError } -func (s *store) PutLayer(id, parent string, names []string, mountLabel string, writeable bool, diff archive.Reader) (*Layer, int64, error) { +func (s *store) PutLayer(id, parent string, names []string, mountLabel string, writeable bool, diff io.Reader) (*Layer, int64, error) { rlstore, err := s.LayerStore() if err != nil { return nil, -1, err @@ -1792,7 +1792,7 @@ func (s *store) Diff(from, to string, options *DiffOptions) (io.ReadCloser, erro return nil, ErrLayerUnknown } -func (s *store) ApplyDiff(to string, diff archive.Reader) (int64, error) { +func (s *store) ApplyDiff(to string, diff io.Reader) (int64, error) { rlstore, err := s.LayerStore() if err != nil { return -1, err