feature: limit blkio device's read/write Bps/IOps

fixes AliyunContainerService#2509

Signed-off-by: Leno Hou <lenohou@gmail.com>

apis/swagger.yml

@@ -2471,12 +2471,6 @@ definitions:
  type: "array"
  items:
  type: "string"
- DiskQuota:
- type: "object"
- description: "update disk quota for container"
- x-nullable: true
- additionalProperties:
- type: "string"
 
  ContainerUpgradeConfig:
  description: |
@@ -2630,6 +2624,12 @@ definitions:
  items:
  type: "string"
  example: "c 13:* rwm"
+ DiskQuota:
+ type: "object"
+ description: "update disk quota for container"
+ x-nullable: true
+ additionalProperties:
+ type: "string"
  KernelMemory:
  description: "Kernel memory limit in bytes."
  type: "integer"

cli/update.go

@@ -51,6 +51,10 @@ func (uc *UpdateCommand) addFlags() {
  flagSet.StringSliceVarP(&uc.labels, "label", "l", nil, "Set label for container")
  flagSet.StringVar(&uc.restartPolicy, "restart", "", "Restart policy to apply when container exits")
  flagSet.StringSliceVar(&uc.diskQuota, "disk-quota", nil, "Update disk quota for container(/=10g)")
+ flagSet.Var(&uc.blkioDeviceReadBps, "device-read-bps", "Update read rate (bytes per second) from a device")
+ flagSet.Var(&uc.blkioDeviceWriteBps, "device-write-bps", "Update write rate (bytes per second) from a device")
+ flagSet.Var(&uc.blkioDeviceReadIOps, "device-read-iops", "Update read rate (io per second) from a device")
+ flagSet.Var(&uc.blkioDeviceWriteIOps, "device-write-iops", "Update write rate (io per second) from a device")
 }
 
 // updateRun is the entry of update command.
@@ -68,23 +72,28 @@ func (uc *UpdateCommand) updateRun(args []string) error {
  return err
  }
 
- resource := types.Resources{
- CPUPeriod: uc.cpuperiod,
- CPUShares: uc.cpushare,
- CPUQuota: uc.cpuquota,
- CpusetCpus: uc.cpusetcpus,
- CpusetMems: uc.cpusetmems,
- Memory: memory,
- MemorySwap: memorySwap,
- BlkioWeight: uc.blkioWeight,
- }
-
- restartPolicy, err := opts.ParseRestartPolicy(uc.restartPolicy)
+ diskQuota, err := opts.ParseDiskQuota(uc.diskQuota)
  if err != nil {
  return err
  }
 
- diskQuota, err := opts.ParseDiskQuota(uc.diskQuota)
+ resource := types.Resources{
+ CPUPeriod: uc.cpuperiod,
+ CPUShares: uc.cpushare,
+ CPUQuota: uc.cpuquota,
+ CpusetCpus: uc.cpusetcpus,
+ CpusetMems: uc.cpusetmems,
+ DiskQuota: diskQuota,
+ Memory: memory,
+ MemorySwap: memorySwap,
+ BlkioWeight: uc.blkioWeight,
+ BlkioDeviceReadBps: uc.blkioDeviceReadBps.Value(),
+ BlkioDeviceWriteBps: uc.blkioDeviceWriteBps.Value(),
+ BlkioDeviceReadIOps: uc.blkioDeviceReadIOps.Value(),
+ BlkioDeviceWriteIOps: uc.blkioDeviceWriteIOps.Value(),
+ }
+
+ restartPolicy, err := opts.ParseRestartPolicy(uc.restartPolicy)
  if err != nil {
  return err
  }
@@ -94,7 +103,6 @@ func (uc *UpdateCommand) updateRun(args []string) error {
  Label: uc.labels,
  RestartPolicy: restartPolicy,
  Resources: resource,
- DiskQuota: diskQuota,
  }
 
  apiClient := uc.cli.Client()

cri/v1alpha2/cri.go

@@ -1105,7 +1105,6 @@ func (c *CriManager) UpdateContainerResources(ctx context.Context, r *runtime.Up
  resources := r.GetLinux()
  updateConfig := &apitypes.UpdateConfig{
  Resources: parseResourcesFromCRI(resources),
- DiskQuota: resources.GetDiskQuota(),
  }
  err = c.ContainerMgr.Update(ctx, containerID, updateConfig)
  if err != nil {

cri/v1alpha2/cri_utils.go

@@ -1043,6 +1043,7 @@ func parseResourcesFromCRI(runtimeResources *runtime.LinuxContainerResources) ap
  BlkioDeviceWriteBps: parseThrottleDeviceFromCRI(runtimeResources.GetBlkioDeviceWriteBps()),
  BlkioDeviceReadIOps: parseThrottleDeviceFromCRI(runtimeResources.GetBlkioDeviceRead_IOps()),
  BlkioDeviceWriteIOps: parseThrottleDeviceFromCRI(runtimeResources.GetBlkioDeviceWrite_IOps()),
+ DiskQuota: runtimeResources.GetDiskQuota(),
  KernelMemory: runtimeResources.GetKernelMemory(),
  MemoryReservation: runtimeResources.GetMemoryReservation(),
  MemorySwappiness: memorySwappiness,

cri/v1alpha2/cri_utils_test.go

@@ -100,6 +100,7 @@ var (
  Memory: 1000,
  CpusetCpus: "0",
  CpusetMems: "0",
+ DiskQuota: map[string]string{"foo": "foo"},
  BlkioWeight: uint16(100),
  BlkioWeightDevice: apitypesWeightDevicesSlice,
  BlkioDeviceReadBps: apitypesThrottleDevicesSlice,

daemon/mgr/container.go

@@ -1048,7 +1048,7 @@ func (mgr *ContainerManager) Update(ctx context.Context, name string, config *ty
  }
 
  // update container disk quota
- if err := mgr.updateContainerDiskQuota(ctx, c, config.DiskQuota); err != nil {
+ if err := mgr.updateContainerDiskQuota(ctx, c, config.Resources.DiskQuota); err != nil {
  return errors.Wrapf(err, "failed to update diskquota of container %s", c.ID)
  }
 

test/cli_run_blkio_test.go

@@ -139,3 +139,103 @@ func (suite *PouchRunBlkioSuite) TestRunWithBlkioWeight(c *check.C) {
  defer DelContainerForceMultyTime(c, name)
  res.Assert(c, icmd.Success)
 }
+
+// TestRunWithBlkIODeviceWriteBps is to verify blkio write bps
+func (suite *PouchRunBlkioSuite) TestRunBlockIODeviceWriteBps(c *check.C) {
+ cname := "TestRunBlockIODeviceWriteBps"
+ testDisk := "/dev/null"
+
+ number, exist := util.GetMajMinNumOfDevice(testDisk)
+ if !exist {
+ c.Skip("fail to get major:minor device number")
+ }
+
+ limitSpeed := "100"
+ expected := fmt.Sprintf("%s:%d %s\n", number, value, limitSpeed)
+
+ blkioDeviceWriteBpsFile := "/sys/fs/cgroup/blkio/blkio.throttle.write_bps_device"
+ throttleDev := testDisk + ":" + limitSpeed
+
+ res := command.PouchRun("run", "--name", cname,
+ "--device-write-bps ", throttleDev, busyboxImage, "cat", blkioDeviceWriteBpsFile)
+ defer DelContainerForceMultyTime(c, cname)
+ res.Assert(c, icmd.Success)
+
+ out := res.Stdout()
+ c.Assert(out, check.Equals, expected)
+}
+
+// TestRunWithBlkIODeviceReadBps is to verify blkio read bps
+func (suite *PouchRunBlkioSuite) TestRunBlockIODeviceReadBps(c *check.C) {
+ cname := "TestRunBlockIODeviceReadBps"
+ testDisk := "/dev/null"
+
+ number, exist := util.GetMajMinNumOfDevice(testDisk)
+ if !exist {
+ c.Skip("fail to get major:minor device number")
+ }
+
+ limitSpeed := "100"
+ expected := fmt.Sprintf("%s:%d %s\n", number, value, limitSpeed)
+
+ blkioDeviceReadBpsFile := "/sys/fs/cgroup/blkio/blkio.throttle.read_bps_device"
+ throttleDev := testDisk + ":" + limitSpeed
+
+ res := command.PouchRun("run", "--name", cname,
+ "--device-read-bps ", throttleDev, busyboxImage, "cat", blkioDeviceReadBpsFile)
+ defer DelContainerForceMultyTime(c, cname)
+ res.Assert(c, icmd.Success)
+
+ out := res.Stdout()
+ c.Assert(out, check.Equals, expected)
+}
+
+// TestRunWithBlkIODeviceWriteIOps is to verify blkio write iops
+func (suite *PouchRunBlkioSuite) TestRunBlockIODeviceWriteIOps(c *check.C) {
+ cname := "TestRunBlockIODeviceWriteIOps"
+ testDisk := "/dev/null"
+
+ number, exist := util.GetMajMinNumOfDevice(testDisk)
+ if !exist {
+ c.Skip("fail to get major:minor device number")
+ }
+
+ limitSpeed := "100"
+ expected := fmt.Sprintf("%s:%d %s\n", number, value, limitSpeed)
+
+ blkioDeviceWriteIOpsFile := "/sys/fs/cgroup/blkio/blkio.throttle.write_iops_device"
+ throttleDev := testDisk + ":" + limitSpeed
+
+ res := command.PouchRun("run", "--name", cname,
+ "--device-write-iops ", throttleDev, busyboxImage, "cat", blkioDeviceWriteIOpsFile)
+ defer DelContainerForceMultyTime(c, cname)
+ res.Assert(c, icmd.Success)
+
+ out := res.Stdout()
+ c.Assert(out, check.Equals, expected)
+}
+
+// TestRunWithBlkIODeviceWriteIOps is to verify blkio read iops
+func (suite *PouchRunBlkioSuite) TestRunBlockIODeviceReadIOps(c *check.C) {
+ cname := "TestRunBlockIODeviceReadIOps"
+ testDisk := "/dev/null"
+
+ number, exist := util.GetMajMinNumOfDevice(testDisk)
+ if !exist {
+ c.Skip("fail to get major:minor device number")
+ }
+
+ limitSpeed := "100"
+ expected := fmt.Sprintf("%s:%d %s\n", number, value, limitSpeed)
+
+ blkioDeviceReadIOpsFile := "/sys/fs/cgroup/blkio/blkio.throttle.read_iops_device"
+ throttleDev := testDisk + ":" + limitSpeed
+
+ res := command.PouchRun("run", "--name", cname,
+ "--device-read-iops ", throttleDev, busyboxImage, "cat", blkioDeviceReadIOpsFile)
+ defer DelContainerForceMultyTime(c, cname)
+ res.Assert(c, icmd.Success)
+
+ out := res.Stdout()
+ c.Assert(out, check.Equals, expected)
+}