diff --git a/apis/opts/diskquota.go b/apis/opts/diskquota.go index 929d549948..88d71b74dc 100644 --- a/apis/opts/diskquota.go +++ b/apis/opts/diskquota.go @@ -17,7 +17,7 @@ func ParseDiskQuota(quotas []string) (map[string]string, error) { parts := strings.Split(quota, "=") switch len(parts) { case 1: - quotaMaps["/"] = parts[0] + quotaMaps[".*"] = parts[0] case 2: quotaMaps[parts[0]] = parts[1] default: diff --git a/apis/opts/diskquota_test.go b/apis/opts/diskquota_test.go index 873bd4e7e9..76bbbe6f7f 100644 --- a/apis/opts/diskquota_test.go +++ b/apis/opts/diskquota_test.go @@ -18,7 +18,7 @@ func TestParseDiskQuota(t *testing.T) { // TODO: Add test cases. {name: "test1", args: args{diskquota: []string{""}}, want: nil, wantErr: true}, {name: "test2", args: args{diskquota: []string{"foo=foo=foo"}}, want: nil, wantErr: true}, - {name: "test3", args: args{diskquota: []string{"foo"}}, want: map[string]string{"/": "foo"}, wantErr: false}, + {name: "test3", args: args{diskquota: []string{"foo"}}, want: map[string]string{".*": "foo"}, wantErr: false}, {name: "test4", args: args{diskquota: []string{"foo=foo"}}, want: map[string]string{"foo": "foo"}, wantErr: false}, {name: "test5", args: args{diskquota: []string{"foo=foo", "bar=bar"}}, want: map[string]string{"foo": "foo", "bar": "bar"}, wantErr: false}, } diff --git a/daemon/mgr/container.go b/daemon/mgr/container.go index 3b7c0c4742..884076e668 100644 --- a/daemon/mgr/container.go +++ b/daemon/mgr/container.go @@ -9,7 +9,6 @@ import ( "os/exec" "path" "path/filepath" - "strconv" "strings" "time" @@ -29,7 +28,6 @@ import ( mountutils "github.com/alibaba/pouch/pkg/mount" "github.com/alibaba/pouch/pkg/streams" "github.com/alibaba/pouch/pkg/utils" - "github.com/alibaba/pouch/storage/quota" volumetypes "github.com/alibaba/pouch/storage/volume/types" "github.com/containerd/cgroups" @@ -358,6 +356,11 @@ func (mgr *ContainerManager) Create(ctx context.Context, name string, config *ty return nil, errors.Wrapf(errtypes.ErrInvalidParam, "NetworkingConfig cannot be empty") } + // validate disk quota + if err := mgr.validateDiskQuota(config); err != nil { + return nil, errors.Wrapf(err, "invalid disk quota config") + } + id, err := mgr.generateContainerID(config.SpecificID) if err != nil { return nil, err @@ -1262,75 +1265,35 @@ func (mgr *ContainerManager) Remove(ctx context.Context, name string, options *t return nil } -func (mgr *ContainerManager) updateContainerDiskQuota(ctx context.Context, c *Container, diskQuota map[string]string) error { +func (mgr *ContainerManager) updateContainerDiskQuota(ctx context.Context, c *Container, diskQuota map[string]string) (err error) { if diskQuota == nil { return nil } + // backup diskquota + origDiskQuota := c.Config.DiskQuota + defer func() { + if err != nil { + c.Lock() + c.Config.DiskQuota = origDiskQuota + c.Unlock() + } + }() + c.Lock() + if c.Config.DiskQuota == nil { + c.Config.DiskQuota = make(map[string]string) + } for dir, quota := range diskQuota { c.Config.DiskQuota[dir] = quota } c.Unlock() // set mount point disk quota - if err := mgr.setMountPointDiskQuota(ctx, c); err != nil { + if err = mgr.setDiskQuota(ctx, c, false); err != nil { return errors.Wrapf(err, "failed to set mount point disk quota") } - c.Lock() - var qid uint32 - if c.Config.QuotaID != "" { - id, err := strconv.Atoi(c.Config.QuotaID) - if err != nil { - return errors.Wrapf(err, "failed to convert QuotaID %s", c.Config.QuotaID) - } - - qid = uint32(id) - if id < 0 { - // QuotaID is < 0, it means pouchd alloc a unique quota id. - qid, err = quota.GetNextQuotaID() - if err != nil { - return errors.Wrap(err, "failed to get next quota id") - } - - // update QuotaID - c.Config.QuotaID = strconv.Itoa(int(qid)) - } - } - c.Unlock() - - // get rootfs quota - defaultQuota := quota.GetDefaultQuota(c.Config.DiskQuota) - if qid > 0 && defaultQuota == "" { - return fmt.Errorf("set quota id but have no set default quota size") - } - // update container rootfs disk quota - // TODO: add lock for container? - rootfs := "" - if c.IsRunningOrPaused() && c.Snapshotter != nil { - basefs, ok := c.Snapshotter.Data["MergedDir"] - if !ok || basefs == "" { - return fmt.Errorf("Container is running, but MergedDir is missing") - } - rootfs = basefs - } else { - if err := mgr.Mount(ctx, c); err != nil { - return errors.Wrapf(err, "failed to mount rootfs: (%s)", c.MountFS) - } - rootfs = c.MountFS - - defer func() { - if err := mgr.Unmount(ctx, c); err != nil { - logrus.Errorf("failed to umount rootfs: (%s), err: (%v)", c.MountFS, err) - } - }() - } - _, err := quota.SetRootfsDiskQuota(rootfs, defaultQuota, qid) - if err != nil { - return errors.Wrapf(err, "failed to set container rootfs diskquota") - } - return nil } diff --git a/daemon/mgr/container_storage.go b/daemon/mgr/container_storage.go index 8e2accdf72..9b0d379988 100644 --- a/daemon/mgr/container_storage.go +++ b/daemon/mgr/container_storage.go @@ -471,52 +471,10 @@ func (mgr *ContainerManager) setMountTab(ctx context.Context, c *Container) erro return nil } -func (mgr *ContainerManager) setRootfsQuota(ctx context.Context, c *Container) error { - logrus.Debugf("start to set rootfs quota, dir(%s)", c.MountFS) +func (mgr *ContainerManager) setDiskQuota(ctx context.Context, c *Container, mounted bool) error { + var globalQuotaID uint32 - if c.MountFS == "" { - return nil - } - - rootfsQuota := quota.GetDefaultQuota(c.Config.DiskQuota) - if rootfsQuota == "" { - return nil - } - - qid := "0" - if c.Config.QuotaID != "" { - qid = c.Config.QuotaID - } - - id, err := strconv.Atoi(qid) - if err != nil { - return errors.Wrapf(err, "failed to change quota id(%s) from string to int", qid) - } - - // set rootfs quota - _, err = quota.SetRootfsDiskQuota(c.MountFS, rootfsQuota, uint32(id)) - if err != nil { - return errors.Wrapf(err, "failed to set rootfs quota, mountfs(%s), quota(%s), quota id(%d)", - c.MountFS, rootfsQuota, id) - } - - return nil -} - -func (mgr *ContainerManager) setMountPointDiskQuota(ctx context.Context, c *Container) error { - if c.Config.DiskQuota == nil { - if c.Config.QuotaID != "" && c.Config.QuotaID != "0" { - return fmt.Errorf("invalid argument, set quota-id without disk-quota") - } - return nil - } - - var ( - qid uint32 - setQuotaID bool - ) - - if c.Config.QuotaID != "" { + if c.Config.QuotaID != "" && c.Config.QuotaID != "0" { id, err := strconv.Atoi(c.Config.QuotaID) if err != nil { return errors.Wrapf(err, "invalid argument, QuotaID(%s)", c.Config.QuotaID) @@ -524,36 +482,41 @@ func (mgr *ContainerManager) setMountPointDiskQuota(ctx context.Context, c *Cont // if QuotaID is < 0, it means pouchd alloc a unique quota id. if id < 0 { - qid, err = quota.GetNextQuotaID() + globalQuotaID, err = quota.GetNextQuotaID() if err != nil { return errors.Wrap(err, "failed to get next quota id") } // update QuotaID - c.Config.QuotaID = strconv.Itoa(int(qid)) + c.Config.QuotaID = strconv.Itoa(int(globalQuotaID)) } else { - qid = uint32(id) + globalQuotaID = uint32(id) } } - if qid > 0 { - setQuotaID = true - } - - // get rootfs quota + // get default quota quotas := c.Config.DiskQuota defaultQuota := quota.GetDefaultQuota(quotas) - if setQuotaID && defaultQuota == "" { - return fmt.Errorf("set quota id but have no set default quota size") - } - // parse diskquota regexe + // parse disk quota regexp var res []*quota.RegExp - for path, size := range quotas { - re := regexp.MustCompile(path) - res = append(res, "a.RegExp{Pattern: re, Path: path, Size: size}) + for key, size := range quotas { + var err error + id := globalQuotaID + if id == 0 { + id, err = quota.GetNextQuotaID() + if err != nil { + return errors.Wrap(err, "failed to get next quota id") + } + } + paths := strings.Split(key, "&") + for _, path := range paths { + re := regexp.MustCompile(path) + res = append(res, "a.RegExp{Pattern: re, Path: path, Size: size, QuotaID: id}) + } } + var mounts []*types.MountPoint for _, mp := range c.Mounts { // skip volume mount or replace mode mount if mp.Replace != "" || mp.Source == "" || mp.Destination == "" { @@ -561,6 +524,7 @@ func (mgr *ContainerManager) setMountPointDiskQuota(ctx context.Context, c *Cont continue } + // skip volume that has set size if mp.Name != "" { v, err := mgr.VolumeMgr.Get(ctx, mp.Name) if err != nil { @@ -580,33 +544,60 @@ func (mgr *ContainerManager) setMountPointDiskQuota(ctx context.Context, c *Cont continue } - matched := false + mounts = append(mounts, mp) + } + + // add rootfs mountpoint + rootfs, err := mgr.getRootfs(ctx, c, mounted) + if err != nil { + return errors.Wrapf(err, "failed to get rootfs") + } + mounts = append(mounts, &types.MountPoint{ + Source: rootfs, + Destination: "/", + }) + + for _, mp := range mounts { + var ( + size string + id uint32 + ) + for _, re := range res { findStr := re.Pattern.FindString(mp.Destination) if findStr == mp.Destination { - quotas[mp.Destination] = re.Size - matched = true + size = re.Size + id = re.QuotaID if re.Path != ".*" { break } } } - size := "" - if matched && !setQuotaID { - size = quotas[mp.Destination] - } else { - size = defaultQuota + if size == "" { + if defaultQuota != "" { + size = defaultQuota + } else { + continue + } } - err := quota.SetDiskQuota(mp.Source, size, qid) - if err != nil { - // just ignore set disk quota fail - logrus.Warnf("failed to set disk quota, directory(%s), size(%s), quotaID(%d), err(%v)", - mp.Source, size, qid, err) + + if mp.Destination == "/" { + // set rootfs quota + _, err = quota.SetRootfsDiskQuota(mp.Source, size, id) + if err != nil { + return errors.Wrapf(err, "failed to set rootfs quota, mountfs(%s), size(%s), quota id(%d)", + mp.Source, size, id) + } + } else { + err := quota.SetDiskQuota(mp.Source, size, id) + if err != nil { + return errors.Wrapf(err, "failed to set disk quota, directory(%s), size(%s), quota id(%d)", + mp.Source, size, id) + } } - } - c.Config.DiskQuota = quotas + } return nil } @@ -699,13 +690,9 @@ func (mgr *ContainerManager) initContainerStorage(ctx context.Context, c *Contai } // set mount point disk quota - if err = mgr.setMountPointDiskQuota(ctx, c); err != nil { - return errors.Wrap(err, "failed to set mount point disk quota") - } - - // set rootfs disk quota - if err = mgr.setRootfsQuota(ctx, c); err != nil { - logrus.Warnf("failed to set rootfs disk quota, err(%v)", err) + if err = mgr.setDiskQuota(ctx, c, true); err != nil { + // just ignore failed to set disk quota + logrus.Warnf("failed to set disk quota, err(%v)", err) } // set volumes into /etc/mtab in container @@ -738,6 +725,35 @@ func (mgr *ContainerManager) SetupWorkingDirectory(ctx context.Context, c *Conta return nil } +func (mgr *ContainerManager) getRootfs(ctx context.Context, c *Container, mounted bool) (string, error) { + var ( + rootfs string + err error + ) + if c.IsRunningOrPaused() && c.Snapshotter != nil { + basefs, ok := c.Snapshotter.Data["MergedDir"] + if !ok || basefs == "" { + return "", fmt.Errorf("Container is running, but MergedDir is missing") + } + rootfs = basefs + } else if !mounted { + if err = mgr.Mount(ctx, c); err != nil { + return "", errors.Wrapf(err, "failed to mount rootfs: (%s)", c.MountFS) + } + rootfs = c.MountFS + + defer func() { + if err = mgr.Unmount(ctx, c); err != nil { + logrus.Errorf("failed to umount rootfs: (%s), err: (%v)", c.MountFS, err) + } + }() + } else { + rootfs = c.MountFS + } + + return rootfs, nil +} + func sortMountPoint(mounts []*types.MountPoint) []*types.MountPoint { sort.Slice(mounts, func(i, j int) bool { if len(mounts[i].Destination) < len(mounts[j].Destination) { diff --git a/daemon/mgr/container_validation.go b/daemon/mgr/container_validation.go index 7339a907e6..53249a49b3 100644 --- a/daemon/mgr/container_validation.go +++ b/daemon/mgr/container_validation.go @@ -3,6 +3,7 @@ package mgr import ( "fmt" "os" + "path/filepath" "strconv" "strings" @@ -28,8 +29,9 @@ var ( // all: all GPUs will be accessible supportedDrivers = map[string]*struct{}{"compute": nil, "compat32": nil, "graphics": nil, "utility": nil, "video": nil, "display": nil} - errInvalidDevice = errors.New("invalid nvidia device") - errInvalidDriver = errors.New("invalid nvidia driver capability") + errInvalidDevice = errors.New("invalid nvidia device") + errInvalidDriver = errors.New("invalid nvidia driver capability") + errInvalidDiskQuota = errors.New("invalid disk quota") // commonLogOpts the option which should be validated in common such as mode, max-buffer-size. commonLogOpts = map[string]bool{ @@ -91,6 +93,46 @@ func (mgr *ContainerManager) validateConfig(c *Container, update bool) ([]string return warnings, nil } +// validateDiskQuota is used to validate disk quota config +func (mgr *ContainerManager) validateDiskQuota(config *types.ContainerCreateConfig) error { + if config == nil { + return errors.Errorf("invalid request, create config is nil") + } + + if config.DiskQuota == nil { + if config.QuotaID != "" && config.QuotaID != "0" { + return errors.Wrap(errInvalidDiskQuota, "set QuotaID without DiskQuota") + } + return nil + } + + quota := config.DiskQuota + if len(quota) > 1 && config.QuotaID != "" { + return errors.Wrap(errInvalidDiskQuota, `QuotaID only used to set one disk quota, `+ + `such as: "/=10G" or "/path1=10G" or ".*=10G"`) + } + + for key := range quota { + if key == "" { + return errors.Wrap(errInvalidDiskQuota, "quota can not be nil string") + } + + paths := strings.Split(key, "&") + if len(paths) <= 1 { + continue + } + + for _, path := range paths { + if !filepath.IsAbs(path) { + return errors.Wrapf(errInvalidDiskQuota, + "(%s) is invalid path in set quota(%s)", path, key) + } + } + } + + return nil +} + // validateRichMode verifies rich mode parameters func validateRichMode(c *Container) error { richModes := []string{ diff --git a/docs/features/pouch_with_diskquota.md b/docs/features/pouch_with_diskquota.md index 34a1319705..decbf834f1 100644 --- a/docs/features/pouch_with_diskquota.md +++ b/docs/features/pouch_with_diskquota.md @@ -10,7 +10,7 @@ hardly do this. Diskquota is designed for limitting filesystem disk usage. Currently PouchContainer supports diskquota which is based on graphdriver overlayfs. Currently in underlying filesystems only ext4 and xfs support diskquota. In -addition, there are three ways to make it: **user quota**, **group quota** and +addition, there are three ways to make it: **group quota** and **project quota**. There are two dimensions to limit disk usage: @@ -25,10 +25,10 @@ PouchContainer only supports block quota now with no inode support temporarily. Diskquota in PouchContainer relies on kernel version PouchContainer runs on. Here is a table describing when each filesystem supports diskquota. -|| user/group quota | project quota| -|:---:| :----:| :---:| -|ext4| >= 2.6|>= 4.5| -|xfs|>= 2.6|>= 3.10| +| | alikernel | open kernel | +| --- | --- | --- | +| ext4 | >= 2.6.32 group quota
>= 4.5 project quota | >= 4.5 project quota | +| xfs (unsupport) | >= 3.10 project quota | >= 3.10 project quota| Although each filesystem in related kernel version supports diskquota, user still needs to install [quota-tools-4.04](https://nchc.dl.sourceforge.net/project/linuxquota/quota-tools/4.04/quota-4.04.tar.gz). @@ -47,25 +47,37 @@ Both two dimensions are covered in PouchContainer's diskquota. ### Parameter Details -Flag `--disk-quota` is used to restrict diskquota of container's corresponding directory. The input type is `string`. +Flag `--disk-quota []string` is used to restrict diskquota of container's corresponding directory. The input type is `string`. -There are three ways to identify the input format: +There are four ways to identify the input format: + +* rule1: `--disk-quota=10GB` : maps container rootfs and all potential volumes binded inside; +* rule2: `--disk-quota=/abc=10GB` : absolute path matching, maps only mount point `/abc` has been limited, container rootfs and any volume haven't been limited; +* rule3: `--disk-quota=/&/abc=10G` : shared size matching, maps container rootfs and mount point `/abc` have been limited, and their total block size are 10G; +* rule4: `--disk-quota=.*=10G`: regular expression matching, maps container rootfs and each mount points have been limited to 10G independently. -* `.*=10GB` maps container rootfs and all potential volumes binded inside; -* `/=10GB` : maps only container rootfs without any volume binded directory in container; -* `/=10GB,/a=10GB`: the front part maps container rootfs, and the back one maps the volume which is binded to directory `/a` inside container. +Flag `--quota-id string` is used to pick an existent quota ID to specify the newly input disk quota. The input type is `string` as well. + +There are three ways to identify the input format: -Flag `--quota-id` is used to pick an existent quota ID to specify the newly input disk quota. The input type is `string` as well. If input `quota-id` is less than 0, pouchd will automatically generate one brand new quotaid and return this ID. If input `quota-id` is 0, pouchd will not set quotaid. If `quota-id` is larger than 0, pouchd will use the input quota ID to set disk quota. +* `--quota-id=-1` : it means pouch daemon will assign a separate available quota id to each mount point, include container rootfs; +* `--quota-id=0` or --quota-id="" or haven't set: it means pouch daemon don't assign quota id; +* `--quota-id=16777216` or more than `16777216`, it means using the specified quota id to set quota. -> valid `quota-id` which is larger than 0 is only used in `upgrade` interface. In this scenario of triggering `upgrade` interface, pouchd will remove the old container and use the new image to take place of old container's image, and create a new container which should inherit the original diskquota. Then user can pass an original `quota-id` of original container to take effect on newly created container. +> 1. `rule1` can't use with `rule2/rule3/rule4` together; +> 2. if it has set quota-id isn't null or `0`, `--disk-quota` only can been set with one rule, or means the length of disk quota slice is `1`. +> 3. `rule3` use `&` link with directory that must be absolute path, can't be regular expression with `rule4`; +> 4. no special characters(`* & ;`) can exist in all mount points +> 5. valid `quota-id` which is larger than `16777216` is only used in `upgrade` interface. In this scenario of triggering `upgrade` interface, pouchd will remove the old container and use the new image to take place of old container's image, and create a new container which should inherit the original diskquota. Then user can pass an original `quota-id` of original container to take effect on newly created container. The effect taken by `disk-quota` and `quota-id` is like the following sheet: | disk-quota | quota-id(<0) | quota-id(=0) | quota-id(>0)| | :--------: | :--------:| :--: |:--: | -| .*=10GB | auto gen quota-id and return，rootfs+n\*volume(total 10GB)|no setting quotaID，rootfs 10GB，each volume 10GB| setting as input quota-id, rootfs+n\*volume(total 10GB) | -| /=10GB | auto gen quota-id and return，only rootfs 10GB)|no setting quotaID，only rootfs 10GB| setting as input quota-id, only rootfs 10GB| -| /=10GB,/a=10GB | invalid |no setting quotaID，rootfs=10GB, only volume mapped to `/a` 10GB| invalid | +| 10GB | auto gen quota-id and return，rootfs+n\*volume(total 10GB) | no setting quotaID，rootfs+n\*volume(total 10GB) | setting as input quota-id, rootfs+n\*volume(total 10GB) | +| /abc=10GB | auto gen quota-id and return，only `/abc` set to 10GB) | no setting quotaID，only `/abc` set to 10GB) | setting as input quota-id, only `/abc` set to 10GB) | +| .*=10GB | auto gen quota-id and return，rootfs+n\*volume(total 10GB) | no setting quotaID，rootfs 10GB，each volume 10GB | setting as input quota-id, rootfs+n\*volume(total 10GB) | +| /&/abc=10GB | auto gen quota-id and return, rootfs+/abc=10GB, another haven't been limited | no setting quotaID，rootfs+/abc=10GB | setting as input quota-id, rootfs+/abc=10GB | Pouchd created local volume with disk quota if user requests to create a volume with size option. If this volume is already set a disk quota rule, then no matter what directory inside container this volume is binded to, and no matter what disk quota user adds to the inside directory again, this volume will be under the original disk quota which is set at the very beginning. diff --git a/storage/quota/quota.go b/storage/quota/quota.go index 436ddcac35..ebf966c6b5 100644 --- a/storage/quota/quota.go +++ b/storage/quota/quota.go @@ -193,16 +193,10 @@ func GetDefaultQuota(quotas map[string]string) string { return "" } - // "/" means the disk quota only takes effect on rootfs + 0 * volume - quota, ok := quotas["/"] - if ok && quota != "" { - return quota - } - // ".*" means the disk quota only takes effect on rootfs + n * volume - quota, ok = quotas[".*"] - if ok && quota != "" { - return quota + size, ok := quotas[".*"] + if ok && size != "" { + return size } return "" diff --git a/storage/quota/quota_test.go b/storage/quota/quota_test.go index 30e3ac7a3a..e91a121a26 100644 --- a/storage/quota/quota_test.go +++ b/storage/quota/quota_test.go @@ -22,7 +22,7 @@ func TestGetDefaultQuota(t *testing.T) { "/": "1000kb", }, }, - want: "1000kb", + want: "", }, { name: "normal case with supposed data .*", @@ -41,7 +41,7 @@ func TestGetDefaultQuota(t *testing.T) { "/": "1000kb", }, }, - want: "1000kb", + want: "2000kb", }, { name: "normal case with no supposed data", diff --git a/storage/quota/type.go b/storage/quota/type.go index 40eebce819..acb4279e29 100644 --- a/storage/quota/type.go +++ b/storage/quota/type.go @@ -7,6 +7,7 @@ type RegExp struct { Pattern *regexp.Regexp Path string Size string + QuotaID uint32 } // OverlayMount represents the parameters of overlay mount.