Skip to content

Commit

Permalink
Merge pull request #2669 from kolyshkin/systemd-unified
Browse files Browse the repository at this point in the history
Initial v2 resources.unified systemd support
  • Loading branch information
Mrunal Patel authored Nov 6, 2020
2 parents e8498d3 + 2e968a8 commit 27227a9
Show file tree
Hide file tree
Showing 16 changed files with 522 additions and 270 deletions.
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -22,5 +22,6 @@ require (
// NOTE: urfave/cli must be <= v1.22.1 due to a regression: https://github.com/urfave/cli/issues/1092
github.com/urfave/cli v1.22.1
github.com/vishvananda/netlink v1.1.0
github.com/willf/bitset v1.1.11
golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f
)
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,8 @@ github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df h1:OviZH7qLw/7Zo
github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243 h1:R43TdZy32XXSXjJn7M/HhALJ9imq6ztLnChfYJpVDnM=
github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
github.com/willf/bitset v1.1.11 h1:N7Z7E9UvjW+sGsEl7k/SJrvY2reP1A07MrGuCjIOjRE=
github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI=
golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
Expand Down
13 changes: 9 additions & 4 deletions libcontainer/cgroups/systemd/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,13 @@ import (
"github.com/sirupsen/logrus"
)

const (
// Default kernel value for cpu quota period is 100000 us (100 ms), same for v1 and v2.
// v1: https://www.kernel.org/doc/html/latest/scheduler/sched-bwc.html and
// v2: https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html
defCPUQuotaPeriod = uint64(100000)
)

var (
connOnce sync.Once
connDbus *systemdDbus.Conn
Expand Down Expand Up @@ -410,10 +417,8 @@ func addCpuQuota(conn *systemdDbus.Conn, properties *[]systemdDbus.Property, quo
cpuQuotaPerSecUSec := uint64(math.MaxUint64)
if quota > 0 {
if period == 0 {
// assume the default kernel value of 100000 us (100 ms), same for v1 and v2.
// v1: https://www.kernel.org/doc/html/latest/scheduler/sched-bwc.html and
// v2: https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html
period = 100000
// assume the default
period = defCPUQuotaPeriod
}
// systemd converts CPUQuotaPerSecUSec (microseconds per CPU second) to CPUQuota
// (integer percentage of CPU) internally. This means that if a fractional percent of
Expand Down
67 changes: 67 additions & 0 deletions libcontainer/cgroups/systemd/cpuset.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
package systemd

import (
"encoding/binary"
"strconv"
"strings"

"github.com/pkg/errors"
"github.com/willf/bitset"
)

// rangeToBits converts a text representation of a CPU mask (as written to
// or read from cgroups' cpuset.* files, e.g. "1,3-5") to a slice of bytes
// with the corresponding bits set (as consumed by systemd over dbus as
// AllowedCPUs/AllowedMemoryNodes unit property value).
func rangeToBits(str string) ([]byte, error) {
bits := &bitset.BitSet{}

for _, r := range strings.Split(str, ",") {
// allow extra spaces around
r = strings.TrimSpace(r)
// allow empty elements (extra commas)
if r == "" {
continue
}
ranges := strings.SplitN(r, "-", 2)
if len(ranges) > 1 {
start, err := strconv.ParseUint(ranges[0], 10, 32)
if err != nil {
return nil, err
}
end, err := strconv.ParseUint(ranges[1], 10, 32)
if err != nil {
return nil, err
}
if start > end {
return nil, errors.New("invalid range: " + r)
}
for i := uint(start); i <= uint(end); i++ {
bits.Set(i)
}
} else {
val, err := strconv.ParseUint(ranges[0], 10, 32)
if err != nil {
return nil, err
}
bits.Set(uint(val))
}
}

val := bits.Bytes()
if len(val) == 0 {
// do not allow empty values
return nil, errors.New("empty value")
}
ret := make([]byte, len(val)*8)
for i := range val {
// bitset uses BigEndian internally
binary.BigEndian.PutUint64(ret[i*8:], val[len(val)-1-i])
}
// remove upper all-zero bytes
for ret[0] == 0 {
ret = ret[1:]
}

return ret, nil
}
58 changes: 58 additions & 0 deletions libcontainer/cgroups/systemd/cpuset_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
package systemd

import (
"bytes"
"testing"
)

func TestRangeToBits(t *testing.T) {
testCases := []struct {
in string
out []byte
isErr bool
}{
{in: "", isErr: true},
{in: "0", out: []byte{1}},
{in: "1", out: []byte{2}},
{in: "0-1", out: []byte{3}},
{in: "0,1", out: []byte{3}},
{in: ",0,1,", out: []byte{3}},
{in: "0-3", out: []byte{0x0f}},
{in: "0,1,2-3", out: []byte{0x0f}},
{in: "4-7", out: []byte{0xf0}},
{in: "0-7", out: []byte{0xff}},
{in: "0-15", out: []byte{0xff, 0xff}},
{in: "16", out: []byte{1, 0, 0}},
{in: "0-3,32-33", out: []byte{3, 0, 0, 0, 0x0f}},
// extra spaces and tabs are ok
{in: "1, 2, 1-2", out: []byte{6}},
{in: " , 1 , 3 , 5-7, ", out: []byte{0xea}},
// somewhat large values
{in: "128-130,1", out: []byte{7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2}},

{in: "-", isErr: true},
{in: "1-", isErr: true},
{in: "-3", isErr: true},
// bad range (start > end)
{in: "54-53", isErr: true},
// kernel does not allow extra spaces inside a range
{in: "1 - 2", isErr: true},
}

for _, tc := range testCases {
t.Logf("case: %q", tc.in)
out, err := rangeToBits(tc.in)
if err != nil {
t.Logf(" got error: %s", err)
if !tc.isErr {
t.Error(" ^^^ unexpected error")
}

continue
}
t.Logf(" expected %v, got %v", tc.out, out)
if !bytes.Equal(out, tc.out) {
t.Error(" ^^^ unexpected result")
}
}
}
130 changes: 130 additions & 0 deletions libcontainer/cgroups/systemd/v2.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
package systemd

import (
"fmt"
"math"
"os"
"path/filepath"
"strconv"
Expand Down Expand Up @@ -34,6 +36,125 @@ func NewUnifiedManager(config *configs.Cgroup, path string, rootless bool) cgrou
}
}

// unifiedResToSystemdProps tries to convert from Cgroup.Resources.Unified
// key/value map (where key is cgroupfs file name) to systemd unit properties.
// This is on a best-effort basis, so the properties that are not known
// (to this function and/or systemd) are ignored (but logged with "debug"
// log level).
//
// For the list of keys, see https://www.kernel.org/doc/Documentation/cgroup-v2.txt
//
// For the list of systemd unit properties, see systemd.resource-control(5).
func unifiedResToSystemdProps(conn *systemdDbus.Conn, res map[string]string) (props []systemdDbus.Property, _ error) {
var err error

for k, v := range res {
if strings.Contains(k, "/") {
return nil, fmt.Errorf("unified resource %q must be a file name (no slashes)", k)
}
sk := strings.SplitN(k, ".", 2)
if len(sk) != 2 {
return nil, fmt.Errorf("unified resource %q must be in the form CONTROLLER.PARAMETER", k)
}
// Kernel is quite forgiving to extra whitespace
// around the value, and so should we.
v = strings.TrimSpace(v)
// Please keep cases in alphabetical order.
switch k {
case "cpu.max":
// value: quota [period]
quota := int64(0) // 0 means "unlimited" for addCpuQuota, if period is set
period := defCPUQuotaPeriod
sv := strings.Fields(v)
if len(sv) < 1 || len(sv) > 2 {
return nil, fmt.Errorf("unified resource %q value invalid: %q", k, v)
}
// quota
if sv[0] != "max" {
quota, err = strconv.ParseInt(sv[0], 10, 64)
if err != nil {
return nil, fmt.Errorf("unified resource %q period value conversion error: %w", k, err)
}
}
// period
if len(sv) == 2 {
period, err = strconv.ParseUint(sv[1], 10, 64)
if err != nil {
return nil, fmt.Errorf("unified resource %q quota value conversion error: %w", k, err)
}
}
addCpuQuota(conn, &props, quota, period)

case "cpu.weight":
num, err := strconv.ParseUint(v, 10, 64)
if err != nil {
return nil, fmt.Errorf("unified resource %q value conversion error: %w", k, err)
}
props = append(props,
newProp("CPUWeight", num))

case "cpuset.cpus", "cpuset.mems":
bits, err := rangeToBits(v)
if err != nil {
return nil, fmt.Errorf("unified resource %q=%q conversion error: %w", k, v, err)
}
m := map[string]string{
"cpuset.cpus": "AllowedCPUs",
"cpuset.mems": "AllowedMemoryNodes",
}
props = append(props,
newProp(m[k], bits))

case "memory.high", "memory.low", "memory.min", "memory.max", "memory.swap.max":
num := uint64(math.MaxUint64)
if v != "max" {
num, err = strconv.ParseUint(v, 10, 64)
if err != nil {
return nil, fmt.Errorf("unified resource %q value conversion error: %w", k, err)
}
}
m := map[string]string{
"memory.high": "MemoryHigh",
"memory.low": "MemoryLow",
"memory.min": "MemoryMin",
"memory.max": "MemoryMax",
"memory.swap.max": "MemorySwapMax",
}
props = append(props,
newProp(m[k], num))

case "pids.max":
num := uint64(math.MaxUint64)
if v != "max" {
var err error
num, err = strconv.ParseUint(v, 10, 64)
if err != nil {
return nil, fmt.Errorf("unified resource %q value conversion error: %w", k, err)
}
}
props = append(props,
newProp("TasksAccounting", true),
newProp("TasksMax", num))

case "memory.oom.group":
// Setting this to 1 is roughly equivalent to OOMPolicy=kill
// (as per systemd.service(5) and
// https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html),
// but it's not clear what to do if it is unset or set
// to 0 in runc update, as there are two other possible
// values for OOMPolicy (continue/stop).
fallthrough

default:
// Ignore the unknown resource here -- will still be
// applied in Set which calls fs2.Set.
logrus.Debugf("don't know how to convert unified resource %q=%q to systemd unit property; skipping (will still be applied to cgroupfs)", k, v)
}
}

return props, nil
}

func genV2ResourcesProperties(c *configs.Cgroup, conn *systemdDbus.Conn) ([]systemdDbus.Property, error) {
var properties []systemdDbus.Property
r := c.Resources
Expand Down Expand Up @@ -82,6 +203,15 @@ func genV2ResourcesProperties(c *configs.Cgroup, conn *systemdDbus.Conn) ([]syst

// ignore r.KernelMemory

// convert Resources.Unified map to systemd properties
if r.Unified != nil {
unifiedProps, err := unifiedResToSystemdProps(conn, r.Unified)
if err != nil {
return nil, err
}
properties = append(properties, unifiedProps...)
}

return properties, nil
}

Expand Down
Loading

0 comments on commit 27227a9

Please sign in to comment.