feature: limit blkio device's read/write Bps/IOps

fixes AliyunContainerService#2509 Signed-off-by: Leno Hou <lenohou@gmail.com>
houstar · Dec 16, 2018 · f6f86de · f6f86de
1 parent 9e879af
commit f6f86de
Show file tree

Hide file tree

Showing 9 changed files with 135 additions and 34 deletions.
diff --git a/apis/swagger.yml b/apis/swagger.yml
@@ -2471,12 +2471,6 @@ definitions:
             type: "array"
             items:
               type: "string"
-          DiskQuota:
-            type: "object"
-            description: "update disk quota for container"
-            x-nullable: true
-            additionalProperties:
-              type: "string"
 
   ContainerUpgradeConfig:
     description: |
@@ -2630,6 +2624,12 @@ definitions:
         items:
           type: "string"
           example: "c 13:* rwm"
+      DiskQuota:
+        type: "object"
+        description: "update disk quota for container"
+        x-nullable: true
+        additionalProperties:
+          type: "string"
       KernelMemory:
         description: "Kernel memory limit in bytes."
         type: "integer"

diff --git a/apis/types/resources.go b/apis/types/resources.go
diff --git a/apis/types/update_config.go b/apis/types/update_config.go
diff --git a/cli/update.go b/cli/update.go
@@ -51,6 +51,10 @@ func (uc *UpdateCommand) addFlags() {
 	flagSet.StringSliceVarP(&uc.labels, "label", "l", nil, "Set label for container")
 	flagSet.StringVar(&uc.restartPolicy, "restart", "", "Restart policy to apply when container exits")
 	flagSet.StringSliceVar(&uc.diskQuota, "disk-quota", nil, "Update disk quota for container(/=10g)")
+	flagSet.Var(&uc.blkioDeviceReadBps, "device-read-bps", "Update read rate (bytes per second) from a device")
+	flagSet.Var(&uc.blkioDeviceWriteBps, "device-write-bps", "Update write rate (bytes per second) from a device")
+	flagSet.Var(&uc.blkioDeviceReadIOps, "device-read-iops", "Update read rate (io per second) from a device")
+	flagSet.Var(&uc.blkioDeviceWriteIOps, "device-write-iops", "Update write rate (io per second) from a device")
 }
 
 // updateRun is the entry of update command.
@@ -68,23 +72,28 @@ func (uc *UpdateCommand) updateRun(args []string) error {
 		return err
 	}
 
-	resource := types.Resources{
-		CPUPeriod:   uc.cpuperiod,
-		CPUShares:   uc.cpushare,
-		CPUQuota:    uc.cpuquota,
-		CpusetCpus:  uc.cpusetcpus,
-		CpusetMems:  uc.cpusetmems,
-		Memory:      memory,
-		MemorySwap:  memorySwap,
-		BlkioWeight: uc.blkioWeight,
-	}
-
-	restartPolicy, err := opts.ParseRestartPolicy(uc.restartPolicy)
+	diskQuota, err := opts.ParseDiskQuota(uc.diskQuota)
 	if err != nil {
 		return err
 	}
 
-	diskQuota, err := opts.ParseDiskQuota(uc.diskQuota)
+	resource := types.Resources{
+		CPUPeriod:            uc.cpuperiod,
+		CPUShares:            uc.cpushare,
+		CPUQuota:             uc.cpuquota,
+		CpusetCpus:           uc.cpusetcpus,
+		CpusetMems:           uc.cpusetmems,
+		DiskQuota:            diskQuota,
+		Memory:               memory,
+		MemorySwap:           memorySwap,
+		BlkioWeight:          uc.blkioWeight,
+		BlkioDeviceReadBps:   uc.blkioDeviceReadBps.Value(),
+		BlkioDeviceWriteBps:  uc.blkioDeviceWriteBps.Value(),
+		BlkioDeviceReadIOps:  uc.blkioDeviceReadIOps.Value(),
+		BlkioDeviceWriteIOps: uc.blkioDeviceWriteIOps.Value(),
+	}
+
+	restartPolicy, err := opts.ParseRestartPolicy(uc.restartPolicy)
 	if err != nil {
 		return err
 	}
@@ -94,7 +103,6 @@ func (uc *UpdateCommand) updateRun(args []string) error {
 		Label:         uc.labels,
 		RestartPolicy: restartPolicy,
 		Resources:     resource,
-		DiskQuota:     diskQuota,
 	}
 
 	apiClient := uc.cli.Client()

diff --git a/cri/v1alpha2/cri.go b/cri/v1alpha2/cri.go
@@ -1105,7 +1105,6 @@ func (c *CriManager) UpdateContainerResources(ctx context.Context, r *runtime.Up
 	resources := r.GetLinux()
 	updateConfig := &apitypes.UpdateConfig{
 		Resources: parseResourcesFromCRI(resources),
-		DiskQuota: resources.GetDiskQuota(),
 	}
 	err = c.ContainerMgr.Update(ctx, containerID, updateConfig)
 	if err != nil {

diff --git a/cri/v1alpha2/cri_utils.go b/cri/v1alpha2/cri_utils.go
@@ -1043,6 +1043,7 @@ func parseResourcesFromCRI(runtimeResources *runtime.LinuxContainerResources) ap
 		BlkioDeviceWriteBps:  parseThrottleDeviceFromCRI(runtimeResources.GetBlkioDeviceWriteBps()),
 		BlkioDeviceReadIOps:  parseThrottleDeviceFromCRI(runtimeResources.GetBlkioDeviceRead_IOps()),
 		BlkioDeviceWriteIOps: parseThrottleDeviceFromCRI(runtimeResources.GetBlkioDeviceWrite_IOps()),
+		DiskQuota:            runtimeResources.GetDiskQuota(),
 		KernelMemory:         runtimeResources.GetKernelMemory(),
 		MemoryReservation:    runtimeResources.GetMemoryReservation(),
 		MemorySwappiness:     memorySwappiness,

diff --git a/cri/v1alpha2/cri_utils_test.go b/cri/v1alpha2/cri_utils_test.go
@@ -100,6 +100,7 @@ var (
 		Memory:             1000,
 		CpusetCpus:         "0",
 		CpusetMems:         "0",
+		DiskQuota:          map[string]string{"foo": "foo"},
 		BlkioWeight:        uint16(100),
 		BlkioWeightDevice:  apitypesWeightDevicesSlice,
 		BlkioDeviceReadBps: apitypesThrottleDevicesSlice,

diff --git a/daemon/mgr/container.go b/daemon/mgr/container.go
@@ -1048,7 +1048,7 @@ func (mgr *ContainerManager) Update(ctx context.Context, name string, config *ty
 	}
 
 	// update container disk quota
-	if err := mgr.updateContainerDiskQuota(ctx, c, config.DiskQuota); err != nil {
+	if err := mgr.updateContainerDiskQuota(ctx, c, config.Resources.DiskQuota); err != nil {
 		return errors.Wrapf(err, "failed to update diskquota of container %s", c.ID)
 	}
 

diff --git a/test/cli_run_blkio_test.go b/test/cli_run_blkio_test.go
@@ -139,3 +139,103 @@ func (suite *PouchRunBlkioSuite) TestRunWithBlkioWeight(c *check.C) {
 	defer DelContainerForceMultyTime(c, name)
 	res.Assert(c, icmd.Success)
 }
+
+// TestRunWithBlkIODeviceWriteBps is to verify blkio write bps
+func (suite *PouchRunBlkioSuite) TestRunBlockIODeviceWriteBps(c *check.C) {
+	cname := "TestRunBlockIODeviceWriteBps"
+	testDisk := "/dev/null"
+
+	number, exist := util.GetMajMinNumOfDevice(testDisk)
+	if !exist {
+		c.Skip("fail to get major:minor device number")
+	}
+
+	limitSpeed := "100"
+	expected := fmt.Sprintf("%s:%d %s\n", number, value, limitSpeed)
+
+	blkioDeviceWriteBpsFile := "/sys/fs/cgroup/blkio/blkio.throttle.write_bps_device"
+	throttleDev := testDisk + ":" + limitSpeed
+
+	res := command.PouchRun("run", "--name", cname,
+		"--device-write-bps ", throttleDev, busyboxImage, "cat", blkioDeviceWriteBps)
+	defer DelContainerForceMultyTime(c, cname)
+	res.Assert(c, icmd.Success)
+
+	out := res.Stdout()
+	c.Assert(out, check.Equals, expected)
+}
+
+// TestRunWithBlkIODeviceReadBps is to verify blkio read bps
+func (suite *PouchRunBlkioSuite) TestRunBlockIODeviceReadBps(c *check.C) {
+	cname := "TestRunBlockIODeviceReadBps"
+	testDisk := "/dev/null"
+
+	number, exist := util.GetMajMinNumOfDevice(testDisk)
+	if !exist {
+		c.Skip("fail to get major:minor device number")
+	}
+
+	limitSpeed := "100"
+	expected := fmt.Sprintf("%s:%d %s\n", number, value, limitSpeed)
+
+	blkioDeviceReadBpsFile := "/sys/fs/cgroup/blkio/blkio.throttle.read_bps_device"
+	throttleDev := testDisk + ":" + limitSpeed
+
+	res := command.PouchRun("run", "--name", cname,
+		"--device-read-bps ", throttleDev, busyboxImage, "cat", blkioDeviceReadBps)
+	defer DelContainerForceMultyTime(c, cname)
+	res.Assert(c, icmd.Success)
+
+	out := res.Stdout()
+	c.Assert(out, check.Equals, expected)
+}
+
+// TestRunWithBlkIODeviceWriteIOps is to verify blkio write iops
+func (suite *PouchRunBlkioSuite) TestRunBlockIODeviceWriteIOps(c *check.C) {
+	cname := "TestRunBlockIODeviceWriteIOps"
+	testDisk := "/dev/null"
+
+	number, exist := util.GetMajMinNumOfDevice(testDisk)
+	if !exist {
+		c.Skip("fail to get major:minor device number")
+	}
+
+	limitSpeed := "100"
+	expected := fmt.Sprintf("%s:%d %s\n", number, value, limitSpeed)
+
+	blkioDeviceWriteIOpsFile := "/sys/fs/cgroup/blkio/blkio.throttle.write_iops_device"
+	throttleDev := testDisk + ":" + limitSpeed
+
+	res := command.PouchRun("run", "--name", cname,
+		"--device-write-iops ", throttleDev, busyboxImage, "cat", blkioDeviceWriteIOpsFile)
+	defer DelContainerForceMultyTime(c, cname)
+	res.Assert(c, icmd.Success)
+
+	out := res.Stdout()
+	c.Assert(out, check.Equals, expected)
+}
+
+// TestRunWithBlkIODeviceWriteIOps is to verify blkio read iops
+func (suite *PouchRunBlkioSuite) TestRunBlockIODeviceReadIOps(c *check.C) {
+	cname := "TestRunBlockIODeviceReadIOps"
+	testDisk := "/dev/null"
+
+	number, exist := util.GetMajMinNumOfDevice(testDisk)
+	if !exist {
+		c.Skip("fail to get major:minor device number")
+	}
+
+	limitSpeed := "100"
+	expected := fmt.Sprintf("%s:%d %s\n", number, value, limitSpeed)
+
+	blkioDeviceReadIOpsFile := "/sys/fs/cgroup/blkio/blkio.throttle.read_iops_device"
+	throttleDev := testDisk + ":" + limitSpeed
+
+	res := command.PouchRun("run", "--name", cname,
+		"--device-read-iops ", throttleDev, busyboxImage, "cat", blkioDeviceReadIOpsFile)
+	defer DelContainerForceMultyTime(c, cname)
+	res.Assert(c, icmd.Success)
+
+	out := res.Stdout()
+	c.Assert(out, check.Equals, expected)
+}