Skip to content

Commit

Permalink
set cgroup parent on build child containers
Browse files Browse the repository at this point in the history
  • Loading branch information
@bparees
bparees committed Jun 16, 2017
1 parent bf857bd commit 16f5632
Show file tree
Hide file tree
Showing 8 changed files with 165 additions and 52 deletions.
16 changes: 11 additions & 5 deletions pkg/build/builder/common.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,10 @@ func execPostCommitHook(client DockerClient, postCommitSpec api.BuildPostCommitS
if err != nil {
return fmt.Errorf("read cgroup limits: %v", err)
}
parent, err := getCgroupParent()
if err != nil {
return fmt.Errorf("read cgroup parent: %v", err)
}

return dockerRun(client, docker.CreateContainerOptions{
Name: containerName,
Expand All @@ -140,11 +144,13 @@ func execPostCommitHook(client DockerClient, postCommitSpec api.BuildPostCommitS
},
HostConfig: &docker.HostConfig{
// Limit container's resource allocation.
CPUShares: limits.CPUShares,
CPUPeriod: limits.CPUPeriod,
CPUQuota: limits.CPUQuota,
Memory: limits.MemoryLimitBytes,
MemorySwap: limits.MemorySwap,
// Though we are capped on memory and cpu at the cgroup parent level,
// some build containers care what their memory limit is so they can
// adapt, thus we need to set the memory limit at the container level
// too, so that information is available to them.
Memory: limits.MemoryLimitBytes,
MemorySwap: limits.MemorySwap,
CgroupParent: parent,
},
}, docker.AttachToContainerOptions{
// Stream logs to stdout and stderr.
Expand Down
9 changes: 6 additions & 3 deletions pkg/build/builder/docker.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -426,13 +426,16 @@ func (d *DockerBuilder) dockerBuild(dir string, tag string, secrets []api.Secret
NetworkMode: string(getDockerNetworkMode()),
}

// Though we are capped on memory and cpu at the cgroup parent level,
// some build containers care what their memory limit is so they can
// adapt, thus we need to set the memory limit at the container level
// too, so that information is available to them.
if d.cgLimits != nil {
opts.Memory = d.cgLimits.MemoryLimitBytes
opts.Memswap = d.cgLimits.MemorySwap
opts.CPUShares = d.cgLimits.CPUShares
opts.CPUPeriod = d.cgLimits.CPUPeriod
opts.CPUQuota = d.cgLimits.CPUQuota
opts.CgroupParent = d.cgLimits.Parent
}

if auth != nil {
opts.AuthConfigs = *auth
}
Expand Down
80 changes: 44 additions & 36 deletions pkg/build/builder/util.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,12 @@ import (
"io/ioutil"
"math"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"

docker "github.com/fsouza/go-dockerclient"
"github.com/opencontainers/runc/libcontainer/cgroups"

s2iapi "github.com/openshift/source-to-image/pkg/api"

Expand Down Expand Up @@ -88,49 +88,21 @@ func GetCGroupLimits() (*s2iapi.CGroupLimits, error) {
byteLimit = 92233720368547
}

// different docker versions seem to use different cgroup directories,
// check for all of them.

// seen on rhel systems
cpuDir := "/sys/fs/cgroup/cpuacct,cpu"

// seen on fedora systems with docker 1.9
// note that in this case there is also a /sys/fs/cgroup/cpu that symlinks
// to /sys/fs/cgroup/cpu,cpuacct, so technically the next check
// would be sufficient, but it seems better to rely on the real directory
// rather than a symlink.
if _, err := os.Stat("/sys/fs/cgroup/cpu,cpuacct"); err == nil {
cpuDir = "/sys/fs/cgroup/cpu,cpuacct"
}

// seen on debian systems with docker 1.10
if _, err := os.Stat("/sys/fs/cgroup/cpu"); err == nil {
cpuDir = "/sys/fs/cgroup/cpu"
}

cpuQuota, err := readInt64(filepath.Join(cpuDir, "cpu.cfs_quota_us"))
if err != nil {
return nil, fmt.Errorf("cannot determine cgroup limits: %v", err)
}

cpuPeriod, err := readInt64(filepath.Join(cpuDir, "cpu.cfs_period_us"))
if err != nil {
return nil, fmt.Errorf("cannot determine cgroup limits: %v", err)
}

cpuShares, err := readInt64(filepath.Join(cpuDir, "cpu.shares"))
parent, err := getCgroupParent()
if err != nil {
return nil, fmt.Errorf("cannot determine cgroup limits: %v", err)
return nil, fmt.Errorf("read cgroup parent: %v", err)
}

return &s2iapi.CGroupLimits{
CPUShares: cpuShares,
CPUPeriod: cpuPeriod,
CPUQuota: cpuQuota,
// Though we are capped on memory and cpu at the cgroup parent level,
// some build containers care what their memory limit is so they can
// adapt, thus we need to set the memory limit at the container level
// too, so that information is available to them.
MemoryLimitBytes: byteLimit,
// Set memoryswap==memorylimit, this ensures no swapping occurs.
// see: https://docs.docker.com/engine/reference/run/#runtime-constraints-on-cpu-and-memory
MemorySwap: byteLimit,
Parent: parent,
}, nil
}

Expand Down Expand Up @@ -201,3 +173,39 @@ func addBuildLabels(labels map[string]string, build *buildapi.Build) {
labels[buildapi.DefaultDockerLabelNamespace+"build.name"] = build.Name
labels[buildapi.DefaultDockerLabelNamespace+"build.namespace"] = build.Namespace
}

// getCgroupParent determines the parent cgroup for a container from
// within that container.
func getCgroupParent() (string, error) {
cgMap, err := cgroups.ParseCgroupFile("/proc/self/cgroup")
if err != nil {
return "", err
}
glog.V(6).Infof("found cgroup values map: %v", cgMap)
return extractParentFromCgroupMap(cgMap)
}

// extractParentFromCgroupMap finds the cgroup parent in the cgroup map
func extractParentFromCgroupMap(cgMap map[string]string) (string, error) {
memory, ok := cgMap["memory"]
if !ok {
return "", fmt.Errorf("could not find memory cgroup subsystem in map %v", cgMap)
}
glog.V(6).Infof("cgroup memory subsystem value: %s", memory)

parts := strings.Split(memory, "/")
if len(parts) < 2 {
return "", fmt.Errorf("unprocessable cgroup memory value: %s", memory)
}

var cgroupParent string
if strings.HasSuffix(memory, ".scope") {
// systemd system, take the second to last segment.
cgroupParent = parts[len(parts)-2]
} else {
// non-systemd, take everything except the last segment.
cgroupParent = strings.Join(parts[:len(parts)-1], "/")
}
glog.V(5).Infof("found cgroup parent %v", cgroupParent)
return cgroupParent, nil
}
104 changes: 104 additions & 0 deletions pkg/build/builder/util_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,3 +97,107 @@ func TestMergeEnv(t *testing.T) {
}
}
}

type testcase struct {
name string
input map[string]string
expect string
fail bool
}

func TestCGroupParentExtraction(t *testing.T) {
tcs := []testcase{
{
name: "systemd",
input: map[string]string{
"cpu": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344.scope",
"cpuacct": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344.scope",
"name=systemd": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344.scope",
"net_prio": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344.scope",
"freezer": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344.scope",
"blkio": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344.scope",
"net_cls": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344.scope",
"memory": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344.scope",
"hugetlb": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344.scope",
"perf_event": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344.scope",
"cpuset": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344.scope",
"devices": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344.scope",
"pids": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344.scope",
},
expect: "kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice",
},
{
name: "systemd-besteffortpod",
input: map[string]string{
"memory": "/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-pod8d369e32_521b_11e7_8df4_507b9d27b5d9.slice/docker-fbf6fe5e4effd80b6a9b3318dd0e5f538b9c4ba8918c174768720c83b338a41f.scope",
},
expect: "kubepods-besteffort-pod8d369e32_521b_11e7_8df4_507b9d27b5d9.slice",
},
{
name: "nonsystemd-burstablepod",
input: map[string]string{
"memory": "/kubepods/burstable/podc4ab0636-521a-11e7-8eea-0e5e65642be0/9ea9361dc31b0e18f699497a5a78a010eb7bae3f9a2d2b5d3027b37bdaa4b334",
},
expect: "/kubepods/burstable/podc4ab0636-521a-11e7-8eea-0e5e65642be0",
},
{
name: "non-systemd",
input: map[string]string{
"cpu": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344",
"cpuacct": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344",
"name=systemd": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344",
"net_prio": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344",
"freezer": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344",
"blkio": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344",
"net_cls": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344",
"memory": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344",
"hugetlb": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344",
"perf_event": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344",
"cpuset": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344",
"devices": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344",
"pids": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344",
},
expect: "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice",
},
{
name: "no-memory-entry",
input: map[string]string{
"cpu": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344",
"cpuacct": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344",
"name=systemd": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344",
"net_prio": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344",
"freezer": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344",
"blkio": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344",
"net_cls": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344",
"hugetlb": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344",
"perf_event": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344",
"cpuset": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344",
"devices": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344",
"pids": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344",
},
expect: "",
fail: true,
},
{
name: "unparseable",
input: map[string]string{
"memory": "kubepods.slice",
},
expect: "",
fail: true,
},
}

for _, tc := range tcs {
parent, err := extractParentFromCgroupMap(tc.input)
if err != nil && !tc.fail {
t.Errorf("[%s] unexpected exception: %v", tc.name, err)
}
if tc.fail && err == nil {
t.Errorf("[%s] expected failure, did not get one and got cgroup parent=%s", tc.name, parent)
}
if parent != tc.expect {
t.Errorf("[%s] expected cgroup parent= %s, got %s", tc.name, tc.expect, parent)
}
}
}
3 changes: 0 additions & 3 deletions test/extended/builds/docker_quota.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,9 +50,6 @@ var _ = g.Describe("[builds][quota][Slow] docker build with a quota", func() {
o.Expect(err).NotTo(o.HaveOccurred())
o.Expect(out).To(o.ContainSubstring("MEMORY=209715200"))
o.Expect(out).To(o.ContainSubstring("MEMORYSWAP=209715200"))
o.Expect(out).To(o.ContainSubstring("SHARES=61"))
o.Expect(out).To(o.ContainSubstring("PERIOD=100000"))
o.Expect(out).To(o.ContainSubstring("QUOTA=6000"))
})
})
})
3 changes: 0 additions & 3 deletions test/extended/builds/s2i_quota.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,9 +52,6 @@ var _ = g.Describe("[builds][Conformance] s2i build with a quota", func() {
o.Expect(err).NotTo(o.HaveOccurred())
o.Expect(buildLog).To(o.ContainSubstring("MEMORY=209715200"))
o.Expect(buildLog).To(o.ContainSubstring("MEMORYSWAP=209715200"))
o.Expect(buildLog).To(o.ContainSubstring("SHARES=61"))
o.Expect(buildLog).To(o.ContainSubstring("PERIOD=100000"))
o.Expect(buildLog).To(o.ContainSubstring("QUOTA=6000"))

events, err := oc.KubeClient().Core().Events(oc.Namespace()).Search(kapi.Scheme, br.Build)
o.Expect(err).NotTo(o.HaveOccurred(), "Should be able to get events from the build")
Expand Down
1 change: 0 additions & 1 deletion test/extended/testdata/bindata.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 0 additions & 1 deletion test/extended/testdata/test-docker-build-quota.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
"spec": {
"resources": {
"limits": {
"cpu": "60m",
"memory": "200Mi"
}
},
Expand Down

0 comments on commit 16f5632

Please sign in to comment.