feature: update blkio device's read/write Bps/IOps

Support update limit of block device, including read/write
bps/iops.

Signed-off-by: Wang Rui <baijia.wr@antfin.com>
Signed-off-by: Leno Hou <lenohou@gmail.com>

cli/update.go

@@ -40,6 +40,10 @@ func (uc *UpdateCommand) addFlags() {
 	flagSet := uc.cmd.Flags()
 	flagSet.SetInterspersed(false)
 	flagSet.Uint16Var(&uc.blkioWeight, "blkio-weight", 0, "Block IO (relative weight), between 10 and 1000, or 0 to disable")
+	flagSet.Var(&uc.blkioDeviceReadBps, "device-read-bps", "Update read rate (bytes per second) from a device")
+	flagSet.Var(&uc.blkioDeviceReadIOps, "device-read-iops", "Update read rate (io per second) from a device")
+	flagSet.Var(&uc.blkioDeviceWriteBps, "device-write-bps", "Update write rate (bytes per second) from a device")
+	flagSet.Var(&uc.blkioDeviceWriteIOps, "device-write-iops", "Update write rate (io per second) from a device")
 	flagSet.Int64Var(&uc.cpuperiod, "cpu-period", 0, "Limit CPU CFS (Completely Fair Scheduler) period, range is in [1000(1ms),1000000(1s)]")
 	flagSet.Int64Var(&uc.cpushare, "cpu-shares", 0, "CPU shares (relative weight)")
 	flagSet.Int64Var(&uc.cpuquota, "cpu-quota", 0, "Limit CPU CFS (Completely Fair Scheduler) quota")
@@ -69,14 +73,18 @@ func (uc *UpdateCommand) updateRun(args []string) error {
 	}
 
 	resource := types.Resources{
-		CPUPeriod:   uc.cpuperiod,
-		CPUShares:   uc.cpushare,
-		CPUQuota:    uc.cpuquota,
-		CpusetCpus:  uc.cpusetcpus,
-		CpusetMems:  uc.cpusetmems,
-		Memory:      memory,
-		MemorySwap:  memorySwap,
-		BlkioWeight: uc.blkioWeight,
+		BlkioWeight:          uc.blkioWeight,
+		BlkioDeviceReadBps:   uc.blkioDeviceReadBps.Value(),
+		BlkioDeviceReadIOps:  uc.blkioDeviceReadIOps.Value(),
+		BlkioDeviceWriteBps:  uc.blkioDeviceWriteBps.Value(),
+		BlkioDeviceWriteIOps: uc.blkioDeviceWriteIOps.Value(),
+		CPUPeriod:            uc.cpuperiod,
+		CPUShares:            uc.cpushare,
+		CPUQuota:             uc.cpuquota,
+		CpusetCpus:           uc.cpusetcpus,
+		CpusetMems:           uc.cpusetmems,
+		Memory:               memory,
+		MemorySwap:           memorySwap,
 	}
 
 	restartPolicy, err := opts.ParseRestartPolicy(uc.restartPolicy)

ctrd/utils.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/alibaba/pouch/apis/types"
 	"github.com/alibaba/pouch/pkg/errtypes"
+	"github.com/alibaba/pouch/pkg/utils"
 
 	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/remotes"
@@ -70,8 +71,28 @@ func toLinuxResources(resources types.Resources) (*specs.LinuxResources, error)
 	r := &specs.LinuxResources{}
 
 	// toLinuxBlockIO
+	readBpsDevice, err := utils.GetThrottleDevice(resources.BlkioDeviceReadBps)
+	if err != nil {
+		return nil, err
+	}
+	readIOpsDevice, err := utils.GetThrottleDevice(resources.BlkioDeviceReadIOps)
+	if err != nil {
+		return nil, err
+	}
+	writeBpsDevice, err := utils.GetThrottleDevice(resources.BlkioDeviceWriteBps)
+	if err != nil {
+		return nil, err
+	}
+	writeIOpsDevice, err := utils.GetThrottleDevice(resources.BlkioDeviceWriteIOps)
+	if err != nil {
+		return nil, err
+	}
 	r.BlockIO = &specs.LinuxBlockIO{
-		Weight: &resources.BlkioWeight,
+		Weight:                  &resources.BlkioWeight,
+		ThrottleReadBpsDevice:   readBpsDevice,
+		ThrottleReadIOPSDevice:  readIOpsDevice,
+		ThrottleWriteBpsDevice:  writeBpsDevice,
+		ThrottleWriteIOPSDevice: writeIOpsDevice,
 	}
 
 	// toLinuxCPU

daemon/mgr/container.go

@@ -1343,6 +1343,18 @@ func (mgr *ContainerManager) updateContainerResources(c *Container, resources ty
 	if resources.BlkioWeight != 0 {
 		cResources.BlkioWeight = resources.BlkioWeight
 	}
+	if len(resources.BlkioDeviceReadBps) != 0 {
+		cResources.BlkioDeviceReadBps = resources.BlkioDeviceReadBps
+	}
+	if len(resources.BlkioDeviceReadIOps) != 0 {
+		cResources.BlkioDeviceReadIOps = resources.BlkioDeviceReadIOps
+	}
+	if len(resources.BlkioDeviceWriteBps) != 0 {
+		cResources.BlkioDeviceWriteBps = resources.BlkioDeviceWriteBps
+	}
+	if len(resources.BlkioDeviceWriteIOps) != 0 {
+		cResources.BlkioDeviceWriteIOps = resources.BlkioDeviceWriteIOps
+	}
 	if resources.CPUPeriod != 0 {
 		cResources.CPUPeriod = resources.CPUPeriod
 	}

daemon/mgr/spec_linux.go

@@ -7,10 +7,10 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
-	"syscall"
 
 	"github.com/alibaba/pouch/apis/opts"
 	"github.com/alibaba/pouch/apis/types"
+	"github.com/alibaba/pouch/pkg/utils"
 
 	"github.com/opencontainers/runc/libcontainer/configs"
 	"github.com/opencontainers/runc/libcontainer/devices"
@@ -129,23 +129,23 @@ func setupResource(ctx context.Context, c *Container, s *specs.Spec) error {
 
 // setupResource creates linux blkio resource spec.
 func setupBlkio(ctx context.Context, r types.Resources, s *specs.Spec) error {
-	weightDevice, err := getWeightDevice(r.BlkioWeightDevice)
+	weightDevice, err := utils.GetWeightDevice(r.BlkioWeightDevice)
 	if err != nil {
 		return err
 	}
-	readBpsDevice, err := getThrottleDevice(r.BlkioDeviceReadBps)
+	readBpsDevice, err := utils.GetThrottleDevice(r.BlkioDeviceReadBps)
 	if err != nil {
 		return err
 	}
-	writeBpsDevice, err := getThrottleDevice(r.BlkioDeviceWriteBps)
+	writeBpsDevice, err := utils.GetThrottleDevice(r.BlkioDeviceWriteBps)
 	if err != nil {
 		return err
 	}
-	readIOpsDevice, err := getThrottleDevice(r.BlkioDeviceReadIOps)
+	readIOpsDevice, err := utils.GetThrottleDevice(r.BlkioDeviceReadIOps)
 	if err != nil {
 		return err
 	}
-	writeIOpsDevice, err := getThrottleDevice(r.BlkioDeviceWriteIOps)
+	writeIOpsDevice, err := utils.GetThrottleDevice(r.BlkioDeviceWriteIOps)
 	if err != nil {
 		return err
 	}
@@ -162,46 +162,6 @@ func setupBlkio(ctx context.Context, r types.Resources, s *specs.Spec) error {
 	return nil
 }
 
-func getWeightDevice(devs []*types.WeightDevice) ([]specs.LinuxWeightDevice, error) {
-	var stat syscall.Stat_t
-	var weightDevice []specs.LinuxWeightDevice
-
-	for _, dev := range devs {
-		if err := syscall.Stat(dev.Path, &stat); err != nil {
-			return nil, err
-		}
-
-		d := specs.LinuxWeightDevice{
-			Weight: &dev.Weight,
-		}
-		d.Major = int64(stat.Rdev >> 8)
-		d.Minor = int64(stat.Rdev & 255)
-		weightDevice = append(weightDevice, d)
-	}
-
-	return weightDevice, nil
-}
-
-func getThrottleDevice(devs []*types.ThrottleDevice) ([]specs.LinuxThrottleDevice, error) {
-	var stat syscall.Stat_t
-	var ThrottleDevice []specs.LinuxThrottleDevice
-
-	for _, dev := range devs {
-		if err := syscall.Stat(dev.Path, &stat); err != nil {
-			return nil, err
-		}
-
-		d := specs.LinuxThrottleDevice{
-			Rate: dev.Rate,
-		}
-		d.Major = int64(stat.Rdev >> 8)
-		d.Minor = int64(stat.Rdev & 255)
-		ThrottleDevice = append(ThrottleDevice, d)
-	}
-
-	return ThrottleDevice, nil
-}
-
 // setupResource creates linux cpu resource spec
 func setupCPU(ctx context.Context, r types.Resources, s *specs.Spec) {
 	cpu := &specs.LinuxCPU{

pkg/utils/resource.go

@@ -0,0 +1,51 @@
+package utils
+
+import (
+	"syscall"
+
+	"github.com/alibaba/pouch/apis/types"
+
+	specs "github.com/opencontainers/runtime-spec/specs-go"
+)
+
+//Convert weight device from []*types.WeightDevice to []specs.LinuxWeightDevice
+func GetWeightDevice(devs []*types.WeightDevice) ([]specs.LinuxWeightDevice, error) {
+	var stat syscall.Stat_t
+	var weightDevice []specs.LinuxWeightDevice
+
+	for _, dev := range devs {
+		if err := syscall.Stat(dev.Path, &stat); err != nil {
+			return nil, err
+		}
+
+		d := specs.LinuxWeightDevice{
+			Weight: &dev.Weight,
+		}
+		d.Major = int64(stat.Rdev >> 8)
+		d.Minor = int64(stat.Rdev & 255)
+		weightDevice = append(weightDevice, d)
+	}
+
+	return weightDevice, nil
+}
+
+// Convert throttle device from []*types.ThrottleDevice to []specs.LinuxThrottleDevice
+func GetThrottleDevice(devs []*types.ThrottleDevice) ([]specs.LinuxThrottleDevice, error) {
+	var stat syscall.Stat_t
+	var ThrottleDevice []specs.LinuxThrottleDevice
+
+	for _, dev := range devs {
+		if err := syscall.Stat(dev.Path, &stat); err != nil {
+			return nil, err
+		}
+
+		d := specs.LinuxThrottleDevice{
+			Rate: dev.Rate,
+		}
+		d.Major = int64(stat.Rdev >> 8)
+		d.Minor = int64(stat.Rdev & 255)
+		ThrottleDevice = append(ThrottleDevice, d)
+	}
+
+	return ThrottleDevice, nil
+}

test/cli_update_test.go

@@ -12,6 +12,7 @@ import (
 	"github.com/alibaba/pouch/pkg/utils"
 	"github.com/alibaba/pouch/test/command"
 	"github.com/alibaba/pouch/test/environment"
+	"github.com/alibaba/pouch/test/util"
 
 	"github.com/go-check/check"
 	"github.com/gotestyourself/gotestyourself/icmd"
@@ -362,7 +363,7 @@ func (suite *PouchUpdateSuite) TestUpdateContainerDeleteEnv(c *check.C) {
 	}
 }
 
-// TestUpdateContainerDiskQuota is to verify the correctness of delete env by update interface
+// TestUpdateContainerDiskQuota is to verify the correctness of disk quota by update interface
 func (suite *PouchUpdateSuite) TestUpdateContainerDiskQuota(c *check.C) {
 	if !environment.IsDiskQuota() {
 		c.Skip("Host does not support disk quota")
@@ -496,3 +497,45 @@ func (suite *PouchUpdateSuite) TestUpdateStoppedContainerCPUQuota(c *check.C) {
 	checkContainerCPUQuota(c, name, "1200")
 
 }
+
+// TestUpdateBlkIOLimit is to verify the correctness of update read/write bps/iops
+func (suite *PouchUpdateSuite) TestUpdateBlkIOLimit(c *check.C) {
+	cname := "TestUpdateBlkIOLimit"
+	testDisk := "/dev/null"
+
+	number, exist := util.GetMajMinNumOfDevice(testDisk)
+	if !exist {
+		c.Skip("fail to get major:minor device number")
+	}
+
+	oldReadBpsDev := testDisk + ":1000"
+	oldReadIopsDev := testDisk + ":1500"
+	oldWriteBpsDev := testDisk + ":2000"
+	oldWriteIopsDev := testDisk + ":2500"
+	newReadBpsDev := testDisk + ":3000"
+	newReadIopsDev := testDisk + ":3500"
+	newWriteBpsDev := testDisk + ":4000"
+	newWriteIopsDev := testDisk + ":4500"
+
+	blkioDeviceReadBpsFile := "/sys/fs/cgroup/blkio/blkio.throttle.read_bps_device"
+	blkioDeviceReadIopsFile := "/sys/fs/cgroup/blkio/blkio.throttle.read_iops_device"
+	blkioDeviceWriteBpsFile := "/sys/fs/cgroup/blkio/blkio.throttle.write_bps_device"
+	blkioDeviceWriteIopsFile := "/sys/fs/cgroup/blkio/blkio.throttle.write_iops_device"
+
+	Expected := fmt.Sprintf("%s %s %s %s %s %s %s %s\n", number, "3000", number, "3500", number, "4000", number, "4500")
+
+	res := command.PouchRun("run", "-d", "--name", cname, "--device-read-bps", oldReadBpsDev, "--device-read-iops", oldReadIopsDev,
+		"--device-write-bps", oldWriteBpsDev, "--device-write-iops", oldWriteIopsDev, busyboxImage, "top").Assert(c, icmd.Success)
+	defer DelContainerForceMultyTime(c, cname)
+
+	command.PouchRun("update", "--device-read-bps", newReadBpsDev, cname).Assert(c, icmd.Success)
+	command.PouchRun("update", "--device-read-iops", newReadIopsDev, cname).Assert(c, icmd.Success)
+	command.PouchRun("update", "--device-write-bps", newWriteBpsDev, cname).Assert(c, icmd.Success)
+	command.PouchRun("update", "--device-write-iops", newWriteIopsDev, cname).Assert(c, icmd.Success)
+
+	// Using "sed" to convert output to one line
+	res = command.PouchRun("exec", cname, "cat", blkioDeviceReadBpsFile, blkioDeviceReadIopsFile, blkioDeviceWriteBpsFile,
+		blkioDeviceWriteIopsFile, "| sed ':a;N;s/\n/ /g;ta'")
+	out := res.Stdout()
+	c.Assert(out, check.Equals, Expected)
+}