From 9842495ef72526a73639e448c6ab77a03276ca47 Mon Sep 17 00:00:00 2001 From: Ace-Tang Date: Tue, 22 May 2018 17:02:59 +0800 Subject: [PATCH] feature: add cgroup resources check check cgroup resource valid in container create, discard un-support cgroup set related flag. Signed-off-by: Ace-Tang --- daemon/mgr/container.go | 11 ++- daemon/mgr/container_utils.go | 138 ++++++++++++++++++++++++++++++++ pkg/system/cgroup.go | 143 ++++++++++++++++++++++++++++++++++ pkg/system/cgroup_test.go | 86 ++++++++++++++++++++ 4 files changed, 376 insertions(+), 2 deletions(-) create mode 100644 pkg/system/cgroup.go create mode 100644 pkg/system/cgroup_test.go diff --git a/daemon/mgr/container.go b/daemon/mgr/container.go index 053da0c39..fefb6ea68 100644 --- a/daemon/mgr/container.go +++ b/daemon/mgr/container.go @@ -360,6 +360,12 @@ func (mgr *ContainerManager) Create(ctx context.Context, name string, config *ty container.Snapshotter.Data["UpperDir"] = upperDir } + // validate container Config + warnings, err := validateConfig(config) + if err != nil { + return nil, err + } + container.Lock() defer container.Unlock() @@ -374,8 +380,9 @@ func (mgr *ContainerManager) Create(ctx context.Context, name string, config *ty mgr.cache.Put(id, container) return &types.ContainerCreateResp{ - ID: id, - Name: name, + ID: id, + Name: name, + Warnings: warnings, }, nil } diff --git a/daemon/mgr/container_utils.go b/daemon/mgr/container_utils.go index e3ea7fb40..dc01bb817 100644 --- a/daemon/mgr/container_utils.go +++ b/daemon/mgr/container_utils.go @@ -10,9 +10,11 @@ import ( "github.com/alibaba/pouch/pkg/errtypes" "github.com/alibaba/pouch/pkg/meta" "github.com/alibaba/pouch/pkg/randomid" + "github.com/alibaba/pouch/pkg/system" "github.com/opencontainers/selinux/go-selinux/label" "github.com/pkg/errors" + "github.com/sirupsen/logrus" ) // containerID returns the container's id, the parameter 'nameOrPrefix' may be container's @@ -223,3 +225,139 @@ func parsePSOutput(output []byte, pids []int) (*types.ContainerProcessList, erro } return procList, nil } + +// validateConfig validates container config +func validateConfig(config *types.ContainerCreateConfig) ([]string, error) { + // validates container hostconfig + warnings := make([]string, 0) + warns, err := validateResource(&config.HostConfig.Resources) + if err != nil { + return nil, err + } + warnings = append(warnings, warns...) + + // TODO: add more validate here + return warnings, nil +} + +func validateResource(r *types.Resources) ([]string, error) { + cgroupInfo := system.NewCgroupInfo() + if cgroupInfo == nil { + return nil, nil + } + warnings := make([]string, 0, 64) + + // validates memory cgroup value + if cgroupInfo.Memory != nil { + if !cgroupInfo.Memory.MemoryLimit { + warn := "Current Kernel does not support memory limit, discard --memory" + logrus.Warn(warn) + warnings = append(warnings, warn) + r.Memory = 0 + } + if !cgroupInfo.Memory.MemorySwap { + warn := "Current Kernel does not support memory swap, discard --memory-swap" + logrus.Warn(warn) + warnings = append(warnings, warn) + r.MemorySwap = -1 + } + if !cgroupInfo.Memory.MemorySwappiness { + warn := "Current Kernel does not support memory swappiness , discard --memory-swappiness" + logrus.Warn(warn) + warnings = append(warnings, warn) + r.MemorySwappiness = nil + } + if !cgroupInfo.Memory.OOMKillDisable { + warn := "Current Kernel does not support disable oom kill, discard --oom-kill-disable" + logrus.Warn(warn) + warnings = append(warnings, warn) + r.OomKillDisable = nil + } + } + + // validates cpu cgroup value + if cgroupInfo.CPU != nil { + if !cgroupInfo.CPU.CpusetCpus { + warn := "Current Kernel does not support cpuset cpus, discard --cpuset-cpus" + logrus.Warn(warn) + warnings = append(warnings, warn) + r.CpusetCpus = "" + } + if !cgroupInfo.CPU.CpusetCpus { + warn := "Current Kernel does not support cpuset cpus, discard --cpuset-mems" + logrus.Warn(warn) + warnings = append(warnings, warn) + r.CpusetMems = "" + } + if !cgroupInfo.CPU.CPUShares { + warn := "Current Kernel does not support cpu shares, discard --cpu-share" + logrus.Warn(warn) + warnings = append(warnings, warn) + r.CPUShares = 0 + } + if !cgroupInfo.CPU.CPUQuota { + warn := "Current Kernel does not support cpu quota, discard --cpu-quota" + logrus.Warn(warn) + warnings = append(warnings, warn) + r.CPUQuota = 0 + } + if !cgroupInfo.CPU.CPUPeriod { + warn := "Current Kernel does not support cpu period, discard --cpu-period" + logrus.Warn(warn) + warnings = append(warnings, warn) + r.CPUPeriod = 0 + } + } + + // validates blkio cgroup value + if cgroupInfo.Blkio != nil { + if !cgroupInfo.Blkio.BlkioWeight { + warn := "Current Kernel does not support blkio weight, discard --blkio-weight" + logrus.Warn(warn) + warnings = append(warnings, warn) + r.BlkioWeight = 0 + } + if !cgroupInfo.Blkio.BlkioWeightDevice { + warn := "Current Kernel does not support blkio weight device, discard --blkio-weight-device" + logrus.Warn(warn) + warnings = append(warnings, warn) + r.BlkioWeightDevice = []*types.WeightDevice{} + } + if !cgroupInfo.Blkio.BlkioDeviceReadBps { + warn := "Current Kernel does not support blkio device throttle read bps, discard --device-read-bps" + logrus.Warn(warn) + warnings = append(warnings, warn) + r.BlkioDeviceReadBps = []*types.ThrottleDevice{} + } + if !cgroupInfo.Blkio.BlkioDeviceWriteBps { + warn := "Current Kernel does not support blkio device throttle write bps, discard --device-write-bps" + logrus.Warn(warn) + warnings = append(warnings, warn) + r.BlkioDeviceWriteBps = []*types.ThrottleDevice{} + } + if !cgroupInfo.Blkio.BlkioDeviceReadIOps { + warn := "Current Kernel does not support blkio device throttle read iops, discard --device-read-iops" + logrus.Warn(warn) + warnings = append(warnings, warn) + r.BlkioDeviceReadIOps = []*types.ThrottleDevice{} + } + if !cgroupInfo.Blkio.BlkioDeviceWriteIOps { + warn := "Current Kernel does not support blkio device throttle, discard --device-write-iops" + logrus.Warn(warn) + warnings = append(warnings, warn) + r.BlkioDeviceWriteIOps = []*types.ThrottleDevice{} + } + } + + // validates pid cgroup value + if cgroupInfo.Pids != nil { + if !cgroupInfo.Pids.Pids { + warn := "Current Kernel does not support pids cgroup, discard --pids-limit" + logrus.Warn(warn) + warnings = append(warnings, warn) + r.PidsLimit = 0 + } + } + + return warnings, nil +} diff --git a/pkg/system/cgroup.go b/pkg/system/cgroup.go new file mode 100644 index 000000000..3ecfe43eb --- /dev/null +++ b/pkg/system/cgroup.go @@ -0,0 +1,143 @@ +package system + +import ( + "bufio" + "os" + "path" + "path/filepath" + "strings" +) + +// MemoryCgroupInfo defines memory cgroup information on current machine +type MemoryCgroupInfo struct { + MemoryLimit bool + MemorySwap bool + MemorySwappiness bool + OOMKillDisable bool +} + +// CPUCgroupInfo defines cpu cgroup information on current machine +type CPUCgroupInfo struct { + CpusetCpus bool + CpusetMems bool + CPUShares bool + CPUPeriod bool + CPUQuota bool +} + +// BlkioCgroupInfo defines blkio cgroup information on current machine +type BlkioCgroupInfo struct { + BlkioWeight bool + BlkioWeightDevice bool + BlkioDeviceReadBps bool + BlkioDeviceWriteBps bool + BlkioDeviceReadIOps bool + BlkioDeviceWriteIOps bool +} + +// PidsCgroupInfo defines pid cgroup information on current machine +type PidsCgroupInfo struct { + Pids bool +} + +// CgroupInfo defines cgroup information on current machine +type CgroupInfo struct { + Memory *MemoryCgroupInfo + CPU *CPUCgroupInfo + Blkio *BlkioCgroupInfo + Pids *PidsCgroupInfo +} + +// NewCgroupInfo news a CgroupInfo struct +func NewCgroupInfo() *CgroupInfo { + cgroupRootPath := getCgroupRootMount("/proc/self/mountinfo") + if cgroupRootPath == "" { + return nil + } + + return &CgroupInfo{ + Memory: getMemoryCgroupInfo(cgroupRootPath), + CPU: getCPUCgroupInfo(cgroupRootPath), + Blkio: getBlkioCgroupInfo(cgroupRootPath), + Pids: getPidsCgroupInfo(cgroupRootPath), + } +} + +func getMemoryCgroupInfo(root string) *MemoryCgroupInfo { + path := path.Join(root, "memory") + return &MemoryCgroupInfo{ + MemoryLimit: isCgroupEnable(path, "memory.limit_in_bytes"), + MemorySwap: isCgroupEnable(path, "memory.memsw.limit_in_bytes"), + MemorySwappiness: isCgroupEnable(path, "memory.swappiness"), + OOMKillDisable: isCgroupEnable(path, "memory.oom_control"), + } +} + +func getCPUCgroupInfo(root string) *CPUCgroupInfo { + cpuPath := path.Join(root, "cpu") + cpusetPath := path.Join(root, "cpuset") + return &CPUCgroupInfo{ + CpusetCpus: isCgroupEnable(cpusetPath, "cpuset.cpus"), + CpusetMems: isCgroupEnable(cpusetPath, "cpuset.mems"), + CPUShares: isCgroupEnable(cpuPath, "cpu.shares"), + CPUQuota: isCgroupEnable(cpuPath, "cpu.cfs_quota_us"), + CPUPeriod: isCgroupEnable(cpuPath, "cpu.cfs_period_us"), + } +} + +func getBlkioCgroupInfo(root string) *BlkioCgroupInfo { + path := path.Join(root, "blkio") + return &BlkioCgroupInfo{ + BlkioWeight: isCgroupEnable(path, "blkio.weight"), + BlkioWeightDevice: isCgroupEnable(path, "blkio.weight_device"), + BlkioDeviceReadBps: isCgroupEnable(path, "blkio.throttle.read_bps_device"), + BlkioDeviceWriteBps: isCgroupEnable(path, "blkio.throttle.write_bps_device"), + BlkioDeviceReadIOps: isCgroupEnable(path, "blkio.throttle.read_iops_device"), + BlkioDeviceWriteIOps: isCgroupEnable(path, "blkio.throttle.write_iops_device"), + } +} + +func getPidsCgroupInfo(root string) *PidsCgroupInfo { + return &PidsCgroupInfo{ + Pids: isCgroupEnable(path.Join(root, "pids")), + } +} + +func isCgroupEnable(f ...string) bool { + _, exist := os.Stat(path.Join(f...)) + return exist == nil +} + +func getCgroupRootMount(mountFile string) string { + f, err := os.Open(mountFile) + if err != nil { + return "" + } + defer f.Close() + + var cgroupRootPath string + scanner := bufio.NewScanner(f) + for scanner.Scan() { + text := scanner.Text() + index := strings.Index(text, " - ") + if index < 0 { + continue + } + fields := strings.Split(text, " ") + postSeparatorFields := strings.Fields(text[index+3:]) + numPostFields := len(postSeparatorFields) + + if len(fields) < 5 || postSeparatorFields[0] != "cgroup" || numPostFields < 3 { + continue + } + + cgroupRootPath = filepath.Dir(fields[4]) + break + } + + if _, err = os.Stat(cgroupRootPath); err != nil { + return "" + } + + return cgroupRootPath +} diff --git a/pkg/system/cgroup_test.go b/pkg/system/cgroup_test.go new file mode 100644 index 000000000..edabfe718 --- /dev/null +++ b/pkg/system/cgroup_test.go @@ -0,0 +1,86 @@ +package system + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestIsCgroupEnable(t *testing.T) { + assert := assert.New(t) + type tCase struct { + exist bool + fs []string + } + + tmpDir, err := ioutil.TempDir("", "test-cgroup-enable") + assert.NoError(err) + defer os.RemoveAll(tmpDir) + + existFile := filepath.Join(tmpDir, "exist") + fd, err := os.Create(existFile) + assert.NoError(err) + fd.Close() + + for _, tc := range []tCase{ + { + exist: false, + fs: []string{tmpDir, "foo"}, + }, + { + exist: false, + fs: []string{tmpDir, "foo", "bar"}, + }, + { + exist: true, + fs: []string{tmpDir, "exist"}, + }, + } { + assert.Equal(tc.exist, isCgroupEnable(tc.fs...)) + } +} + +func TestGetCgroupRootMount(t *testing.T) { + assert := assert.New(t) + type tCase struct { + cgroupMount string + data string + } + + tmpDir, err := ioutil.TempDir("", "test-cgroup-enable") + assert.NoError(err) + defer os.RemoveAll(tmpDir) + file := filepath.Join(tmpDir, "mountinfo") + + path := filepath.Join(tmpDir, "cgroup") + fd, err := os.Create(path) + fd.Close() + assert.NoError(err) + + for _, tc := range []tCase{ + { + cgroupMount: "", + data: "", + }, + { + cgroupMount: "", + data: "a b\n a b", + }, + { + cgroupMount: "", + data: "18 58 0:17 / /sys rw,relatime shared:6 - sysfs sysfs rw", + }, + { + cgroupMount: tmpDir, + data: fmt.Sprintf("a a a a %s a a - cgroup a a\n a b c", path), + }, + } { + err := ioutil.WriteFile(file, []byte(tc.data), 0644) + assert.NoError(err) + assert.Equal(tc.cgroupMount, getCgroupRootMount(file)) + } +}