Skip to content
This repository has been archived by the owner on Feb 8, 2023. It is now read-only.

Commit

Permalink
feature: support updating block io limit
Browse files Browse the repository at this point in the history 
support updating blkio read/write bps/iops.

Signed-off-by: Wang Rui <baijia.wr@antfin.com>
  • Loading branch information
@zjumoon01
zjumoon01 committed Jan 7, 2019
1 parent f8f73f6 commit d70f016
Show file tree
Hide file tree
Showing 2 changed files with 203 additions and 4 deletions.
100 changes: 97 additions & 3 deletions tests/integration/update.bats
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,30 @@ function check_cgroup_value() {
[ "$status" -eq 0 ]
check_cgroup_value $CGROUP_BLKIO "blkio.weight" 500

blk_deivce=$(lsblk | grep [sv]da | grep disk | awk '{print $2}')
blk_major=$(echo $blk_deivce | awk -F ':' '{print $1}')
blk_minor=$(echo $blk_deivce | awk -F ':' '{print $2}')

# update device-read-bps
runc update test_update --device-read-bps "$blk_deivce 1048576"
[ "$status" -eq 0 ]
check_cgroup_value $CGROUP_BLKIO "blkio.throttle.read_bps_device" "$blk_major:$blk_minor 1048576"

# update device-read-iops
runc update test_update --device-read-iops "$blk_deivce 2000"
[ "$status" -eq 0 ]
check_cgroup_value $CGROUP_BLKIO "blkio.throttle.read_iops_device" "$blk_major:$blk_minor 2000"

# update device-write-bps
runc update test_update --device-write-bps "$blk_deivce 2097152"
[ "$status" -eq 0 ]
check_cgroup_value $CGROUP_BLKIO "blkio.throttle.write_bps_device" "$blk_major:$blk_minor 2097152"

# update device-write-iops
runc update test_update --device-write-iops "$blk_deivce 3000"
[ "$status" -eq 0 ]
check_cgroup_value $CGROUP_BLKIO "blkio.throttle.write_iops_device" "$blk_major:$blk_minor 3000"

# update cpu-period
runc update test_update --cpu-period 900000
[ "$status" -eq 0 ]
Expand Down Expand Up @@ -183,7 +207,35 @@ function check_cgroup_value() {
"cpus": "0"
},
"blockIO": {
"weight": 1000
"weight": 1000,
"throttleReadBpsDevice": [
{
"major": $blk_major,
"minor": $blk_minor,
"rate": 0
}
],
"throttleReadIOPSDevice": [
{
"major": $blk_major,
"minor": $blk_minor,
"rate": 0
}
],
"throttleWriteBpsDevice": [
{
"major": $blk_major,
"minor": $blk_minor,
"rate": 0
}
],
"throttleWriteIOPSDevice": [
{
"major": $blk_major,
"minor": $blk_minor,
"rate": 0
}
]
},
"pids": {
"limit": 20
Expand All @@ -192,6 +244,10 @@ function check_cgroup_value() {
EOF
[ "$status" -eq 0 ]
check_cgroup_value $CGROUP_BLKIO "blkio.weight" 1000
check_cgroup_value $CGROUP_BLKIO "blkio.throttle.read_bps_device" ""
check_cgroup_value $CGROUP_BLKIO "blkio.throttle.read_iops_device" ""
check_cgroup_value $CGROUP_BLKIO "blkio.throttle.write_bps_device" ""
check_cgroup_value $CGROUP_BLKIO "blkio.throttle.write_iops_device" ""
check_cgroup_value $CGROUP_CPU "cpu.cfs_period_us" 1000000
check_cgroup_value $CGROUP_CPU "cpu.cfs_quota_us" 500000
check_cgroup_value $CGROUP_CPU "cpu.shares" 100
Expand All @@ -204,11 +260,17 @@ EOF

# redo all the changes at once
runc update test_update --blkio-weight 500 \
--device-read-bps "$blk_deivce 1048576" --device-read-iops "$blk_deivce 2000" \
--device-write-bps "$blk_deivce 2097152" --device-write-iops "$blk_deivce 3000" \
--cpu-period 900000 --cpu-quota 600000 --cpu-share 200 --memory 67108864 \
--memory-reservation 33554432 --kernel-memory 50331648 --kernel-memory-tcp 41943040 \
--pids-limit 10
[ "$status" -eq 0 ]
check_cgroup_value $CGROUP_BLKIO "blkio.weight" 500
check_cgroup_value $CGROUP_BLKIO "blkio.throttle.read_bps_device" "$blk_major:$blk_minor 1048576"
check_cgroup_value $CGROUP_BLKIO "blkio.throttle.read_iops_device" "$blk_major:$blk_minor 2000"
check_cgroup_value $CGROUP_BLKIO "blkio.throttle.write_bps_device" "$blk_major:$blk_minor 2097152"
check_cgroup_value $CGROUP_BLKIO "blkio.throttle.write_iops_device" "$blk_major:$blk_minor 3000"
check_cgroup_value $CGROUP_CPU "cpu.cfs_period_us" 900000
check_cgroup_value $CGROUP_CPU "cpu.cfs_quota_us" 600000
check_cgroup_value $CGROUP_CPU "cpu.shares" 200
Expand All @@ -219,7 +281,7 @@ EOF
check_cgroup_value $CGROUP_PIDS "pids.max" 10

# reset to initial test value via json file
DATA=$(cat <<"EOF"
DATA=$(cat <<EOF
{
"memory": {
"limit": 33554432,
Expand All @@ -234,7 +296,35 @@ EOF
"cpus": "0"
},
"blockIO": {
"weight": 1000
"weight": 1000,
"throttleReadBpsDevice": [
{
"major": $blk_major,
"minor": $blk_minor,
"rate": 0
}
],
"throttleReadIOPSDevice": [
{
"major": $blk_major,
"minor": $blk_minor,
"rate": 0
}
],
"throttleWriteBpsDevice": [
{
"major": $blk_major,
"minor": $blk_minor,
"rate": 0
}
],
"throttleWriteIOPSDevice": [
{
"major": $blk_major,
"minor": $blk_minor,
"rate": 0
}
]
},
"pids": {
"limit": 20
Expand All @@ -247,6 +337,10 @@ EOF
runc update -r $BATS_TMPDIR/runc-cgroups-integration-test.json test_update
[ "$status" -eq 0 ]
check_cgroup_value $CGROUP_BLKIO "blkio.weight" 1000
check_cgroup_value $CGROUP_BLKIO "blkio.throttle.read_bps_device" ""
check_cgroup_value $CGROUP_BLKIO "blkio.throttle.read_iops_device" ""
check_cgroup_value $CGROUP_BLKIO "blkio.throttle.write_bps_device" ""
check_cgroup_value $CGROUP_BLKIO "blkio.throttle.write_iops_device" ""
check_cgroup_value $CGROUP_CPU "cpu.cfs_period_us" 1000000
check_cgroup_value $CGROUP_CPU "cpu.cfs_quota_us" 500000
check_cgroup_value $CGROUP_CPU "cpu.shares" 100
Expand Down
107 changes: 106 additions & 1 deletion update.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"fmt"
"os"
"strconv"
"strings"

"github.com/docker/go-units"
"github.com/opencontainers/runc/libcontainer/configs"
Expand Down Expand Up @@ -49,7 +50,27 @@ The accepted format is as follow (unchanged values can be omitted):
"mems": ""
},
"blockIO": {
"weight": 0
"weight": 0,
"throttleReadBpsDevice": [{
"major",
"minor",
"rate"
}],
"throttleWriteBpsDevice": [{
"major",
"minor",
"rate"
}],
"throttleReadIOPSDevice": [{
"major",
"minor",
"rate"
}],
"throttleWriteIOPSDevice": [{
"major",
"minor",
"rate"
}]
}
}
Expand All @@ -62,6 +83,22 @@ other options are ignored.
Name: "blkio-weight",
Usage: "Specifies per cgroup weight, range is from 10 to 1000",
},
cli.StringSliceFlag{
Name: "device-read-bps",
Usage: "Limit read rate (bytes per second) from a device, such as \"8:0 1048576\"",
},
cli.StringSliceFlag{
Name: "device-write-bps",
Usage: "Limit write rate (bytes per second) to a device, such as \"8:0 1048576\"",
},
cli.StringSliceFlag{
Name: "device-read-iops",
Usage: "Limit read rate (IO per second) from a device, such as \"8:0 5000\"",
},
cli.StringSliceFlag{
Name: "device-write-iops",
Usage: "Limit write rate (IO per second) to a device, such as \"8:0 5000\"",
},
cli.StringFlag{
Name: "cpu-period",
Usage: "CPU CFS period to be used for hardcapping (in usecs). 0 to use system default",
Expand Down Expand Up @@ -181,6 +218,23 @@ other options are ignored.
if val := context.Int("blkio-weight"); val != 0 {
r.BlockIO.Weight = u16Ptr(uint16(val))
}
for _, pair := range []struct {
opt string
dest *[]specs.LinuxThrottleDevice
}{
{"device-read-bps", &r.BlockIO.ThrottleReadBpsDevice},
{"device-write-bps", &r.BlockIO.ThrottleWriteBpsDevice},
{"device-read-iops", &r.BlockIO.ThrottleReadIOPSDevice},
{"device-write-iops", &r.BlockIO.ThrottleWriteIOPSDevice},
} {
if val := context.StringSlice(pair.opt); len(val) > 0 {
var err error
*pair.dest, err = stringSliceToThrottleDevice(val)
if err != nil {
return fmt.Errorf("invalid value for %s: %s", pair.opt, err)
}
}
}
if val := context.String("cpuset-cpus"); val != "" {
r.CPU.Cpus = val
}
Expand Down Expand Up @@ -264,6 +318,20 @@ other options are ignored.
config.Cgroups.Resources.MemorySwap = *r.Memory.Swap
config.Cgroups.Resources.PidsLimit = r.Pids.Limit

for _, pair := range []struct {
dest *[]*configs.ThrottleDevice
src []specs.LinuxThrottleDevice
}{
{&config.Cgroups.Resources.BlkioThrottleReadBpsDevice, r.BlockIO.ThrottleReadBpsDevice},
{&config.Cgroups.Resources.BlkioThrottleWriteBpsDevice, r.BlockIO.ThrottleWriteBpsDevice},
{&config.Cgroups.Resources.BlkioThrottleReadIOPSDevice, r.BlockIO.ThrottleReadIOPSDevice},
{&config.Cgroups.Resources.BlkioThrottleWriteIOPSDevice, r.BlockIO.ThrottleWriteIOPSDevice},
} {
for _, td := range pair.src {
*pair.dest = append(*pair.dest, configs.NewThrottleDevice(td.Major, td.Minor, td.Rate))
}
}

// Update Intel RDT
l3CacheSchema := context.String("l3-cache-schema")
memBwSchema := context.String("mem-bw-schema")
Expand Down Expand Up @@ -302,3 +370,40 @@ other options are ignored.
return container.Set(config)
},
}

func stringSliceToThrottleDevice(ss []string) ([]specs.LinuxThrottleDevice, error) {
tds := make([]specs.LinuxThrottleDevice, 0, len(ss))
for _, v := range ss {
parts := strings.SplitN(strings.TrimSpace(v), " ", 2)
if len(parts) != 2 {
return nil, fmt.Errorf("failed to parse throttle device %s: the expected format is 'major:minor rate'", v)
}
devs := strings.SplitN(parts[0], ":", 2)
if len(devs) != 2 {
return nil, fmt.Errorf("failed to parse throttle device %s: the expected format is 'major:minor rate'", v)
}

td := specs.LinuxThrottleDevice{}
for _, item := range []struct {
value string
dest *int64
}{
{devs[0], &td.Major},
{devs[1], &td.Minor},
} {
v, err := strconv.ParseInt(item.value, 10, 64)
if err != nil {
return nil, err
}
*item.dest = v
}
rate, err := strconv.ParseUint(parts[1], 10, 64)
if err != nil {
return nil, err
}
td.Rate = rate

tds = append(tds, td)
}
return tds, nil
}

0 comments on commit d70f016

Please sign in to comment.