diff --git a/cgroups/cgroups.go b/cgroups/cgroups.go new file mode 100644 index 000000000..e108f3106 --- /dev/null +++ b/cgroups/cgroups.go @@ -0,0 +1,104 @@ +package cgroups + +import ( + "bufio" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + + rspec "github.com/opencontainers/runtime-spec/specs-go" +) + +var ( + // AbsCgroupPath is absolute path for container's cgroup mount + AbsCgroupPath = "/cgrouptest" + // RelCgroupPath is relative path for container's cgroup mount + RelCgroupPath = "testdir/cgrouptest/container" +) + +// Cgroup represents interfaces for cgroup validation +type Cgroup interface { + GetBlockIOData(pid int, cgPath string) (*rspec.LinuxBlockIO, error) + GetCPUData(pid int, cgPath string) (*rspec.LinuxCPU, error) + GetDevicesData(pid int, cgPath string) ([]rspec.LinuxDeviceCgroup, error) + GetHugepageLimitData(pid int, cgPath string) ([]rspec.LinuxHugepageLimit, error) + GetMemoryData(pid int, cgPath string) (*rspec.LinuxMemory, error) + GetNetworkData(pid int, cgPath string) (*rspec.LinuxNetwork, error) + GetPidsData(pid int, cgPath string) (*rspec.LinuxPids, error) +} + +// FindCgroup gets cgroup root mountpoint +func FindCgroup() (Cgroup, error) { + f, err := os.Open("/proc/self/mountinfo") + if err != nil { + return nil, err + } + defer f.Close() + + cgroupv2 := false + scanner := bufio.NewScanner(f) + for scanner.Scan() { + text := scanner.Text() + fields := strings.Split(text, " ") + // Safe as mountinfo encodes mountpoints with spaces as \040. + index := strings.Index(text, " - ") + postSeparatorFields := strings.Fields(text[index+3:]) + numPostFields := len(postSeparatorFields) + + // This is an error as we can't detect if the mount is for "cgroup" + if numPostFields == 0 { + return nil, fmt.Errorf("Found no fields post '-' in %q", text) + } + + if postSeparatorFields[0] == "cgroup" { + // Check that the mount is properly formated. + if numPostFields < 3 { + return nil, fmt.Errorf("Error found less than 3 fields post '-' in %q", text) + } + + cg := &CgroupV1{ + MountPath: filepath.Dir(fields[4]), + } + return cg, nil + } else if postSeparatorFields[0] == "cgroup2" { + cgroupv2 = true + continue + //TODO cgroupv2 unimplemented + } + } + + if err := scanner.Err(); err != nil { + return nil, err + } + + if cgroupv2 { + return nil, fmt.Errorf("cgroupv2 is not supported yet") + } + return nil, fmt.Errorf("cgroup is not found") +} + +// GetSubsystemPath gets path of subsystem +func GetSubsystemPath(pid int, subsystem string) (string, error) { + contents, err := ioutil.ReadFile(fmt.Sprintf("/proc/%d/cgroup", pid)) + if err != nil { + return "", err + } + + parts := strings.Split(strings.TrimSpace(string(contents)), "\n") + for _, part := range parts { + elem := strings.SplitN(part, ":", 3) + if len(elem) < 3 { + continue + } + subelems := strings.Split(elem[1], ",") + for _, subelem := range subelems { + if subelem == subsystem { + return elem[2], nil + } + } + } + + return "", fmt.Errorf("subsystem %s not found", subsystem) +} diff --git a/cgroups/cgroups_v1.go b/cgroups/cgroups_v1.go new file mode 100644 index 000000000..3f056821c --- /dev/null +++ b/cgroups/cgroups_v1.go @@ -0,0 +1,553 @@ +package cgroups + +import ( + "fmt" + "io/ioutil" + "path/filepath" + "regexp" + "strconv" + "strings" + + rspec "github.com/opencontainers/runtime-spec/specs-go" +) + +// CgroupV1 used for cgroupv1 validation +type CgroupV1 struct { + MountPath string +} + +func getDeviceID(id string) (int64, int64, error) { + elem := strings.Split(id, ":") + major, err := strconv.ParseInt(elem[0], 10, 64) + if err != nil { + return 0, 0, err + } + minor, err := strconv.ParseInt(elem[1], 10, 64) + if err != nil { + return 0, 0, err + } + return major, minor, nil +} + +// GetBlockIOData gets cgroup blockio data +func (cg *CgroupV1) GetBlockIOData(pid int, cgPath string) (*rspec.LinuxBlockIO, error) { + lb := &rspec.LinuxBlockIO{} + names := []string{"weight", "leaf_weight", "weight_device", "leaf_weight_device", "throttle.read_bps_device", "throttle.write_bps_device", "throttle.read_iops_device", "throttle.write_iops_device"} + for i, name := range names { + fileName := strings.Join([]string{"blkio", name}, ".") + filePath := filepath.Join(cg.MountPath, "blkio", cgPath, fileName) + if !filepath.IsAbs(cgPath) { + subPath, err := GetSubsystemPath(pid, "blkio") + if err != nil { + return nil, err + } + if !strings.Contains(subPath, RelCgroupPath) { + return nil, fmt.Errorf("cgroup subsystem %s is not mounted as expected", "blkio") + } + filePath = filepath.Join(cg.MountPath, "blkio", subPath, fileName) + } + contents, err := ioutil.ReadFile(filePath) + if err != nil { + return nil, err + } + switch i { + case 0: + res, err := strconv.ParseUint(strings.TrimSpace(string(contents)), 10, 16) + if err != nil { + return nil, err + } + weight := uint16(res) + lb.Weight = &weight + case 1: + res, err := strconv.ParseUint(strings.TrimSpace(string(contents)), 10, 16) + if err != nil { + return nil, err + } + leafWeight := uint16(res) + lb.LeafWeight = &leafWeight + case 2: + parts := strings.Split(strings.TrimSpace(string(contents)), "\n") + for _, part := range parts { + elem := strings.Split(part, " ") + major, minor, err := getDeviceID(elem[0]) + if err != nil { + return nil, err + } + res, err := strconv.ParseUint(elem[1], 10, 16) + if err != nil { + return nil, err + } + weight := uint16(res) + lwd := rspec.LinuxWeightDevice{} + lwd.Major = major + lwd.Minor = minor + lwd.Weight = &weight + lb.WeightDevice = append(lb.WeightDevice, lwd) + } + case 3: + parts := strings.Split(strings.TrimSpace(string(contents)), "\n") + for _, part := range parts { + elem := strings.Split(part, " ") + major, minor, err := getDeviceID(elem[0]) + if err != nil { + return nil, err + } + res, err := strconv.ParseUint(elem[1], 10, 16) + if err != nil { + return nil, err + } + leafWeight := uint16(res) + exist := false + for i, wd := range lb.WeightDevice { + if wd.Major == major && wd.Minor == minor { + exist = true + lb.WeightDevice[i].LeafWeight = &leafWeight + break + } + } + if !exist { + lwd := rspec.LinuxWeightDevice{} + lwd.Major = major + lwd.Minor = minor + lwd.LeafWeight = &leafWeight + lb.WeightDevice = append(lb.WeightDevice, lwd) + } + } + case 4: + parts := strings.Split(strings.TrimSpace(string(contents)), "\n") + for _, part := range parts { + elem := strings.Split(part, " ") + major, minor, err := getDeviceID(elem[0]) + if err != nil { + return nil, err + } + rate, err := strconv.ParseUint(elem[1], 10, 64) + if err != nil { + return nil, err + } + ltd := rspec.LinuxThrottleDevice{} + ltd.Major = major + ltd.Minor = minor + ltd.Rate = rate + lb.ThrottleReadBpsDevice = append(lb.ThrottleReadBpsDevice, ltd) + } + case 5: + parts := strings.Split(strings.TrimSpace(string(contents)), "\n") + for _, part := range parts { + elem := strings.Split(part, " ") + major, minor, err := getDeviceID(elem[0]) + if err != nil { + return nil, err + } + rate, err := strconv.ParseUint(elem[1], 10, 64) + if err != nil { + return nil, err + } + ltd := rspec.LinuxThrottleDevice{} + ltd.Major = major + ltd.Minor = minor + ltd.Rate = rate + lb.ThrottleWriteBpsDevice = append(lb.ThrottleWriteBpsDevice, ltd) + } + case 6: + parts := strings.Split(strings.TrimSpace(string(contents)), "\n") + for _, part := range parts { + elem := strings.Split(part, " ") + major, minor, err := getDeviceID(elem[0]) + if err != nil { + return nil, err + } + rate, err := strconv.ParseUint(elem[1], 10, 64) + if err != nil { + return nil, err + } + ltd := rspec.LinuxThrottleDevice{} + ltd.Major = major + ltd.Minor = minor + ltd.Rate = rate + lb.ThrottleReadIOPSDevice = append(lb.ThrottleReadIOPSDevice, ltd) + } + case 7: + parts := strings.Split(strings.TrimSpace(string(contents)), "\n") + for _, part := range parts { + elem := strings.Split(part, " ") + major, minor, err := getDeviceID(elem[0]) + if err != nil { + return nil, err + } + rate, err := strconv.ParseUint(elem[1], 10, 64) + if err != nil { + return nil, err + } + ltd := rspec.LinuxThrottleDevice{} + ltd.Major = major + ltd.Minor = minor + ltd.Rate = rate + lb.ThrottleWriteIOPSDevice = append(lb.ThrottleWriteIOPSDevice, ltd) + } + } + } + + return lb, nil +} + +// GetCPUData gets cgroup cpus data +func (cg *CgroupV1) GetCPUData(pid int, cgPath string) (*rspec.LinuxCPU, error) { + lc := &rspec.LinuxCPU{} + names := []string{"shares", "cfs_quota_us", "cfs_period_us"} + for i, name := range names { + fileName := strings.Join([]string{"cpu", name}, ".") + filePath := filepath.Join(cg.MountPath, "cpu", cgPath, fileName) + if !filepath.IsAbs(cgPath) { + subPath, err := GetSubsystemPath(pid, "cpu") + if err != nil { + return nil, err + } + if !strings.Contains(subPath, RelCgroupPath) { + return nil, fmt.Errorf("cgroup subsystem %s is not mounted as expected", "cpu") + } + filePath = filepath.Join(cg.MountPath, "cpu", subPath, fileName) + } + contents, err := ioutil.ReadFile(filePath) + if err != nil { + return nil, err + } + switch i { + case 0: + res, err := strconv.ParseUint(strings.TrimSpace(string(contents)), 10, 64) + if err != nil { + return nil, err + } + shares := res + lc.Shares = &shares + case 1: + res, err := strconv.ParseInt(strings.TrimSpace(string(contents)), 10, 64) + if err != nil { + return nil, err + } + quota := res + lc.Quota = "a + case 2: + res, err := strconv.ParseUint(strings.TrimSpace(string(contents)), 10, 64) + if err != nil { + return nil, err + } + period := res + lc.Period = &period + } + } + // CONFIG_RT_GROUP_SCHED may be not set + // Can always get rt data from /proc + contents, err := ioutil.ReadFile("/proc/sys/kernel/sched_rt_period_us") + if err != nil { + return nil, err + } + rtPeriod, err := strconv.ParseUint(strings.TrimSpace(string(contents)), 10, 64) + if err != nil { + return nil, err + } + lc.RealtimePeriod = &rtPeriod + contents, err = ioutil.ReadFile("/proc/sys/kernel/sched_rt_runtime_us") + if err != nil { + return nil, err + } + rtQuota, err := strconv.ParseInt(strings.TrimSpace(string(contents)), 10, 64) + if err != nil { + return nil, err + } + lc.RealtimeRuntime = &rtQuota + + names = []string{"cpus", "mems"} + for i, name := range names { + fileName := strings.Join([]string{"cpuset", name}, ".") + filePath := filepath.Join(cg.MountPath, "cpuset", cgPath, fileName) + if !filepath.IsAbs(cgPath) { + subPath, err := GetSubsystemPath(pid, "cpuset") + if err != nil { + return nil, err + } + if !strings.Contains(subPath, RelCgroupPath) { + return nil, fmt.Errorf("cgroup subsystem %s is not mounted as expected", "cpuset") + } + filePath = filepath.Join(cg.MountPath, "cpuset", subPath, fileName) + } + contents, err := ioutil.ReadFile(filePath) + if err != nil { + return nil, err + } + switch i { + case 0: + lc.Cpus = strings.TrimSpace(string(contents)) + case 1: + lc.Mems = strings.TrimSpace(string(contents)) + } + } + + return lc, nil +} + +// GetDevicesData gets cgroup devices data +func (cg *CgroupV1) GetDevicesData(pid int, cgPath string) ([]rspec.LinuxDeviceCgroup, error) { + ld := []rspec.LinuxDeviceCgroup{} + + return ld, nil +} + +func inBytes(size string) (int64, error) { + KiB := 1024 + MiB := 1024 * KiB + GiB := 1024 * MiB + TiB := 1024 * GiB + PiB := 1024 * TiB + binaryMap := map[string]int64{"k": int64(KiB), "m": int64(MiB), "g": int64(GiB), "t": int64(TiB), "p": int64(PiB)} + sizeRegex := regexp.MustCompile(`^(\d+(\.\d+)*) ?([kKmMgGtTpP])?[bB]?$`) + matches := sizeRegex.FindStringSubmatch(size) + if len(matches) != 4 { + return -1, fmt.Errorf("invalid size: '%s'", size) + } + + byteSize, err := strconv.ParseFloat(matches[1], 64) + if err != nil { + return -1, err + } + + unitPrefix := strings.ToLower(matches[3]) + if mul, ok := binaryMap[unitPrefix]; ok { + byteSize *= float64(mul) + } + + return int64(byteSize), nil +} + +func getHugePageSize() ([]string, error) { + var pageSizes []string + sizeList := []string{"B", "kB", "MB", "GB", "TB", "PB"} + files, err := ioutil.ReadDir("/sys/kernel/mm/hugepages") + if err != nil { + return pageSizes, err + } + for _, st := range files { + nameArray := strings.Split(st.Name(), "-") + pageSize, err := inBytes(nameArray[1]) + if err != nil { + return []string{}, err + } + size := float64(pageSize) + base := float64(1024.0) + i := 0 + unitsLimit := len(sizeList) - 1 + for size >= base && i < unitsLimit { + size = size / base + i++ + } + sizeString := fmt.Sprintf("%g%s", size, sizeList[i]) + pageSizes = append(pageSizes, sizeString) + } + + return pageSizes, nil +} + +// GetHugepageLimitData gets cgroup hugetlb data +func (cg *CgroupV1) GetHugepageLimitData(pid int, cgPath string) ([]rspec.LinuxHugepageLimit, error) { + lh := []rspec.LinuxHugepageLimit{} + pageSizes, err := getHugePageSize() + if err != nil { + return lh, err + } + for _, pageSize := range pageSizes { + maxUsage := strings.Join([]string{"hugetlb", pageSize, "limit_in_bytes"}, ".") + filePath := filepath.Join(cg.MountPath, "hugetlb", cgPath, maxUsage) + if !filepath.IsAbs(cgPath) { + subPath, err := GetSubsystemPath(pid, "hugetlb") + if err != nil { + return lh, err + } + if !strings.Contains(subPath, RelCgroupPath) { + return nil, fmt.Errorf("cgroup subsystem %s is not mounted as expected", "hugetlb") + } + filePath = filepath.Join(cg.MountPath, "hugetlb", subPath, maxUsage) + } + contents, err := ioutil.ReadFile(filePath) + if err != nil { + return lh, err + } + res, err := strconv.ParseUint(strings.TrimSpace(string(contents)), 10, 64) + if err != nil { + return nil, err + } + pageLimit := rspec.LinuxHugepageLimit{} + pageLimit.Pagesize = pageSize + pageLimit.Limit = res + lh = append(lh, pageLimit) + } + + return lh, nil +} + +// GetMemoryData gets cgroup memory data +func (cg *CgroupV1) GetMemoryData(pid int, cgPath string) (*rspec.LinuxMemory, error) { + lm := &rspec.LinuxMemory{} + names := []string{"limit_in_bytes", "soft_limit_in_bytes", "memsw.limit_in_bytes", "kmem.limit_in_bytes", "kmem.tcp.limit_in_bytes", "swappiness", "oom_control"} + for i, name := range names { + fileName := strings.Join([]string{"memory", name}, ".") + filePath := filepath.Join(cg.MountPath, "memory", cgPath, fileName) + if !filepath.IsAbs(cgPath) { + subPath, err := GetSubsystemPath(pid, "memory") + if err != nil { + return nil, err + } + if !strings.Contains(subPath, RelCgroupPath) { + return nil, fmt.Errorf("cgroup subsystem %s is not mounted as expected", "memory") + } + filePath = filepath.Join(cg.MountPath, "memory", subPath, fileName) + } + contents, err := ioutil.ReadFile(filePath) + if err != nil { + return nil, err + } + switch i { + case 0: + res, err := strconv.ParseInt(strings.TrimSpace(string(contents)), 10, 64) + if err != nil { + return nil, err + } + limit := res + lm.Limit = &limit + case 1: + res, err := strconv.ParseInt(strings.TrimSpace(string(contents)), 10, 64) + if err != nil { + return nil, err + } + sLimit := res + lm.Reservation = &sLimit + case 2: + res, err := strconv.ParseInt(strings.TrimSpace(string(contents)), 10, 64) + if err != nil { + return nil, err + } + swLimit := res + lm.Swap = &swLimit + case 3: + res, err := strconv.ParseInt(strings.TrimSpace(string(contents)), 10, 64) + if err != nil { + return nil, err + } + kernelLimit := res + lm.Kernel = &kernelLimit + case 4: + res, err := strconv.ParseInt(strings.TrimSpace(string(contents)), 10, 64) + if err != nil { + return nil, err + } + tcpLimit := res + lm.KernelTCP = &tcpLimit + case 5: + res, err := strconv.ParseUint(strings.TrimSpace(string(contents)), 10, 64) + if err != nil { + return nil, err + } + swappiness := res + lm.Swappiness = &swappiness + case 6: + parts := strings.Split(string(contents), "\n") + part := strings.Split(parts[0], " ") + res, err := strconv.ParseInt(part[1], 10, 64) + if err != nil { + return nil, err + } + oom := false + if res == 1 { + oom = true + } + lm.DisableOOMKiller = &oom + } + } + + return lm, nil +} + +// GetNetworkData gets cgroup network data +func (cg *CgroupV1) GetNetworkData(pid int, cgPath string) (*rspec.LinuxNetwork, error) { + ln := &rspec.LinuxNetwork{} + fileName := strings.Join([]string{"net_cls", "classid"}, ".") + filePath := filepath.Join(cg.MountPath, "net_cls", cgPath, fileName) + if !filepath.IsAbs(cgPath) { + subPath, err := GetSubsystemPath(pid, "net_cls") + if err != nil { + return nil, err + } + if !strings.Contains(subPath, RelCgroupPath) { + return nil, fmt.Errorf("cgroup subsystem %s is not mounted as expected", "net_cls") + } + filePath = filepath.Join(cg.MountPath, "net_cls", subPath, fileName) + } + contents, err := ioutil.ReadFile(filePath) + if err != nil { + return nil, err + } + res, err := strconv.ParseUint(strings.TrimSpace(string(contents)), 10, 64) + if err != nil { + return nil, err + } + classid := uint32(res) + ln.ClassID = &classid + + fileName = strings.Join([]string{"net_prio", "ifpriomap"}, ".") + filePath = filepath.Join(cg.MountPath, "net_prio", cgPath, fileName) + if !filepath.IsAbs(cgPath) { + subPath, err := GetSubsystemPath(pid, "net_prio") + if err != nil { + return nil, err + } + if !strings.Contains(subPath, RelCgroupPath) { + return nil, fmt.Errorf("cgroup subsystem %s is not mounted as expected", "net_prio") + } + filePath = filepath.Join(cg.MountPath, "net_prio", subPath, fileName) + } + contents, err = ioutil.ReadFile(filePath) + if err != nil { + return nil, err + } + parts := strings.Split(strings.TrimSpace(string(contents)), "\n") + for _, part := range parts { + elem := strings.Split(part, " ") + res, err := strconv.ParseUint(elem[1], 10, 64) + if err != nil { + return nil, err + } + lip := rspec.LinuxInterfacePriority{} + lip.Name = elem[0] + lip.Priority = uint32(res) + ln.Priorities = append(ln.Priorities, lip) + } + + return ln, nil +} + +// GetPidsData gets cgroup pids data +func (cg *CgroupV1) GetPidsData(pid int, cgPath string) (*rspec.LinuxPids, error) { + lp := &rspec.LinuxPids{} + fileName := strings.Join([]string{"pids", "max"}, ".") + filePath := filepath.Join(cg.MountPath, "pids", cgPath, fileName) + if !filepath.IsAbs(cgPath) { + subPath, err := GetSubsystemPath(pid, "pids") + if err != nil { + return nil, err + } + if !strings.Contains(subPath, RelCgroupPath) { + return nil, fmt.Errorf("cgroup subsystem %s is not mounted as expected", "pids") + } + filePath = filepath.Join(cg.MountPath, "pids", subPath, fileName) + } + contents, err := ioutil.ReadFile(filePath) + if err != nil { + return nil, err + } + res, err := strconv.ParseInt(strings.TrimSpace(string(contents)), 10, 64) + if err != nil { + return nil, err + } + lp.Limit = res + + return lp, nil +} diff --git a/cgroups/cgroups_v2.go b/cgroups/cgroups_v2.go new file mode 100644 index 000000000..d15e647d9 --- /dev/null +++ b/cgroups/cgroups_v2.go @@ -0,0 +1,47 @@ +package cgroups + +import ( + "fmt" + + rspec "github.com/opencontainers/runtime-spec/specs-go" +) + +// CgroupV2 used for cgroupv2 validation +type CgroupV2 struct { + MountPath string +} + +// GetBlockIOData gets cgroup blockio data +func GetBlockIOData(pid int, cgPath string) (*rspec.LinuxBlockIO, error) { + return nil, fmt.Errorf("unimplemented yet") +} + +// GetCPUData gets cgroup cpus data +func GetCPUData(pid int, cgPath string) (*rspec.LinuxCPU, error) { + return nil, fmt.Errorf("unimplemented yet") +} + +// GetDevicesData gets cgroup devices data +func GetDevicesData(pid int, cgPath string) ([]rspec.LinuxDeviceCgroup, error) { + return nil, fmt.Errorf("unimplemented yet") +} + +// GetHugepageLimitData gets cgroup hugetlb data +func GetHugepageLimitData(pid int, cgPath string) ([]rspec.LinuxHugepageLimit, error) { + return nil, fmt.Errorf("unimplemented yet") +} + +// GetMemoryData gets cgroup memory data +func (cg *CgroupV2) GetMemoryData(pid int, cgPath string) (*rspec.LinuxMemory, error) { + return nil, fmt.Errorf("unimplemented yet") +} + +// GetNetworkData gets cgroup network data +func GetNetworkData(pid int, cgPath string) (*rspec.LinuxNetwork, error) { + return nil, fmt.Errorf("unimplemented yet") +} + +// GetPidsData gets cgroup pid ints data +func GetPidsData(pid int, cgPath string) (*rspec.LinuxPids, error) { + return nil, fmt.Errorf("unimplemented yet") +} diff --git a/validation/linux_cgroups_blkio.go b/validation/linux_cgroups_blkio.go new file mode 100644 index 000000000..40ae1d016 --- /dev/null +++ b/validation/linux_cgroups_blkio.go @@ -0,0 +1,115 @@ +package main + +import ( + "fmt" + + "github.com/opencontainers/runtime-tools/cgroups" + "github.com/opencontainers/runtime-tools/validation/util" +) + +func main() { + var weight uint16 = 500 + var leafWeight uint16 = 300 + var major, minor int64 = 8, 0 + var rate uint64 = 102400 + g := util.GetDefaultGenerator() + g.SetLinuxCgroupsPath(cgroups.AbsCgroupPath) + g.SetLinuxResourcesBlockIOWeight(weight) + g.SetLinuxResourcesBlockIOLeafWeight(leafWeight) + g.AddLinuxResourcesBlockIOWeightDevice(major, minor, weight) + g.AddLinuxResourcesBlockIOLeafWeightDevice(major, minor, leafWeight) + g.AddLinuxResourcesBlockIOThrottleReadBpsDevice(major, minor, rate) + g.AddLinuxResourcesBlockIOThrottleWriteBpsDevice(major, minor, rate) + g.AddLinuxResourcesBlockIOThrottleReadIOPSDevice(major, minor, rate) + g.AddLinuxResourcesBlockIOThrottleWriteIOPSDevice(major, minor, rate) + err := util.RuntimeOutsideValidate(g, cgroups.AbsCgroupPath, func(pid int, path string) error { + cg, err := cgroups.FindCgroup() + if err != nil { + return err + } + lbd, err := cg.GetBlockIOData(pid, path) + if err != nil { + return err + } + if *lbd.Weight != weight { + return fmt.Errorf("blkio weight is not set correctly, expect: %d, actual: %d", weight, lbd.Weight) + } + if *lbd.LeafWeight != leafWeight { + return fmt.Errorf("blkio leafWeight is not set correctly, expect: %d, actual: %d", weight, lbd.LeafWeight) + } + + found := false + for _, wd := range lbd.WeightDevice { + if wd.Major == major && wd.Minor == minor { + found = true + if *wd.Weight != weight { + return fmt.Errorf("blkio weight for %d:%d is not set correctly, expect: %d, actual: %d", major, minor, weight, wd.Weight) + } + if *wd.LeafWeight != leafWeight { + return fmt.Errorf("blkio leafWeight for %d:%d is not set correctly, expect: %d, actual: %d", major, minor, leafWeight, wd.LeafWeight) + } + } + } + if !found { + return fmt.Errorf("blkio weightDevice for %d:%d is not set", major, minor) + } + + found = false + for _, trbd := range lbd.ThrottleReadBpsDevice { + if trbd.Major == major && trbd.Minor == minor { + found = true + if trbd.Rate != rate { + return fmt.Errorf("blkio read bps for %d:%d is not set correctly, expect: %d, actual: %d", major, minor, rate, trbd.Rate) + } + } + } + if !found { + return fmt.Errorf("blkio read bps for %d:%d is not set", major, minor) + } + + found = false + for _, twbd := range lbd.ThrottleWriteBpsDevice { + if twbd.Major == major && twbd.Minor == minor { + found = true + if twbd.Rate != rate { + return fmt.Errorf("blkio write bps for %d:%d is not set correctly, expect: %d, actual: %d", major, minor, rate, twbd.Rate) + } + } + } + if !found { + return fmt.Errorf("blkio write bps for %d:%d is not set", major, minor) + } + + found = false + for _, trid := range lbd.ThrottleReadIOPSDevice { + if trid.Major == major && trid.Minor == minor { + found = true + if trid.Rate != rate { + return fmt.Errorf("blkio read iops for %d:%d is not set correctly, expect: %d, actual: %d", major, minor, rate, trid.Rate) + } + } + } + if !found { + return fmt.Errorf("blkio read iops for %d:%d is not set", major, minor) + } + + found = false + for _, twid := range lbd.ThrottleWriteIOPSDevice { + if twid.Major == major && twid.Minor == minor { + found = true + if twid.Rate != rate { + return fmt.Errorf("blkio write iops for %d:%d is not set correctly, expect: %d, actual: %d", major, minor, rate, twid.Rate) + } + } + } + if !found { + return fmt.Errorf("blkio write iops for %d:%d is not set", major, minor) + } + + return nil + }) + + if err != nil { + util.Fatal(err) + } +} diff --git a/validation/linux_cgroups_cpus.go b/validation/linux_cgroups_cpus.go new file mode 100644 index 000000000..32728a205 --- /dev/null +++ b/validation/linux_cgroups_cpus.go @@ -0,0 +1,52 @@ +package main + +import ( + "fmt" + + "github.com/opencontainers/runtime-tools/cgroups" + "github.com/opencontainers/runtime-tools/validation/util" +) + +func main() { + var shares uint64 = 1024 + var period uint64 = 100000 + var quota int64 = 50000 + var cpus, mems string = "0-1", "0" + g := util.GetDefaultGenerator() + g.SetLinuxCgroupsPath(cgroups.AbsCgroupPath) + g.SetLinuxResourcesCPUShares(shares) + g.SetLinuxResourcesCPUQuota(quota) + g.SetLinuxResourcesCPUPeriod(period) + g.SetLinuxResourcesCPUCpus(cpus) + g.SetLinuxResourcesCPUMems(mems) + err := util.RuntimeOutsideValidate(g, cgroups.AbsCgroupPath, func(pid int, path string) error { + cg, err := cgroups.FindCgroup() + if err != nil { + return err + } + lcd, err := cg.GetCPUData(pid, path) + if err != nil { + return err + } + if *lcd.Shares != shares { + return fmt.Errorf("cpus shares limit is not set correctly, expect: %d, actual: %d", shares, lcd.Shares) + } + if *lcd.Quota != quota { + return fmt.Errorf("cpus quota is not set correctly, expect: %d, actual: %d", quota, lcd.Quota) + } + if *lcd.Period != period { + return fmt.Errorf("cpus period is not set correctly, expect: %d, actual: %d", period, lcd.Period) + } + if lcd.Cpus != cpus { + return fmt.Errorf("cpus cpus is not set correctly, expect: %s, actual: %s", cpus, lcd.Cpus) + } + if lcd.Mems != mems { + return fmt.Errorf("cpus mems is not set correctly, expect: %s, actual: %s", mems, lcd.Mems) + } + return nil + }) + + if err != nil { + util.Fatal(err) + } +} diff --git a/validation/linux_cgroups_hugetlb.go b/validation/linux_cgroups_hugetlb.go new file mode 100644 index 000000000..ae1b81d09 --- /dev/null +++ b/validation/linux_cgroups_hugetlb.go @@ -0,0 +1,35 @@ +package main + +import ( + "fmt" + + "github.com/opencontainers/runtime-tools/cgroups" + "github.com/opencontainers/runtime-tools/validation/util" +) + +func main() { + page := "1GB" + var limit uint64 = 56892210544640 + g := util.GetDefaultGenerator() + g.SetLinuxCgroupsPath(cgroups.AbsCgroupPath) + g.AddLinuxResourcesHugepageLimit(page, limit) + err := util.RuntimeOutsideValidate(g, cgroups.AbsCgroupPath, func(pid int, path string) error { + cg, err := cgroups.FindCgroup() + if err != nil { + return err + } + lhd, err := cg.GetHugepageLimitData(pid, path) + if err != nil { + return err + } + for _, lhl := range lhd { + if lhl.Pagesize == page && lhl.Limit != limit { + return fmt.Errorf("hugepage %s limit is not set correctly, expect: %d, actual: %d", page, limit, lhl.Limit) + } + } + return nil + }) + if err != nil { + util.Fatal(err) + } +} diff --git a/validation/linux_cgroups_memory.go b/validation/linux_cgroups_memory.go new file mode 100644 index 000000000..7293d6388 --- /dev/null +++ b/validation/linux_cgroups_memory.go @@ -0,0 +1,57 @@ +package main + +import ( + "fmt" + + "github.com/opencontainers/runtime-tools/cgroups" + "github.com/opencontainers/runtime-tools/validation/util" +) + +func main() { + var limit int64 = 50593792 + var swappiness uint64 = 50 + g := util.GetDefaultGenerator() + g.SetLinuxCgroupsPath(cgroups.AbsCgroupPath) + g.SetLinuxResourcesMemoryLimit(limit) + g.SetLinuxResourcesMemoryReservation(limit) + g.SetLinuxResourcesMemorySwap(limit) + g.SetLinuxResourcesMemoryKernel(limit) + g.SetLinuxResourcesMemoryKernelTCP(limit) + g.SetLinuxResourcesMemorySwappiness(swappiness) + g.SetLinuxResourcesMemoryDisableOOMKiller(true) + err := util.RuntimeOutsideValidate(g, cgroups.AbsCgroupPath, func(pid int, path string) error { + cg, err := cgroups.FindCgroup() + if err != nil { + return err + } + lm, err := cg.GetMemoryData(pid, path) + if err != nil { + return err + } + if limit != *lm.Limit { + return fmt.Errorf("memory limit is not set correctly, expect: %d, actual: %d", limit, *lm.Limit) + } + if limit != *lm.Reservation { + return fmt.Errorf("memory reservation is not set correctly, expect: %d, actual: %d", limit, *lm.Reservation) + } + if limit != *lm.Swap { + return fmt.Errorf("memory swap is not set correctly, expect: %d, actual: %d", limit, *lm.Reservation) + } + if limit != *lm.Kernel { + return fmt.Errorf("memory kernel is not set correctly, expect: %d, actual: %d", limit, *lm.Kernel) + } + if limit != *lm.KernelTCP { + return fmt.Errorf("memory kernelTCP is not set correctly, expect: %d, actual: %d", limit, *lm.Kernel) + } + if swappiness != *lm.Swappiness { + return fmt.Errorf("memory swappiness is not set correctly, expect: %d, actual: %d", swappiness, *lm.Swappiness) + } + if true != *lm.DisableOOMKiller { + return fmt.Errorf("memory oom is not set correctly, expect: %t, actual: %t", true, *lm.DisableOOMKiller) + } + return nil + }) + if err != nil { + util.Fatal(err) + } +} diff --git a/validation/linux_cgroups_network.go b/validation/linux_cgroups_network.go new file mode 100644 index 000000000..9cbd0d59b --- /dev/null +++ b/validation/linux_cgroups_network.go @@ -0,0 +1,47 @@ +package main + +import ( + "fmt" + + "github.com/opencontainers/runtime-tools/cgroups" + "github.com/opencontainers/runtime-tools/validation/util" +) + +func main() { + var id, prio uint32 = 255, 10 + ifName := "lo" + g := util.GetDefaultGenerator() + g.SetLinuxCgroupsPath(cgroups.AbsCgroupPath) + g.SetLinuxResourcesNetworkClassID(id) + err := util.RuntimeOutsideValidate(g, cgroups.AbsCgroupPath, func(pid int, path string) error { + cg, err := cgroups.FindCgroup() + if err != nil { + return err + } + lnd, err := cg.GetNetworkData(pid, path) + if err != nil { + return err + } + if *lnd.ClassID != id { + return fmt.Errorf("network ID is not set correctly, expect: %d, actual: %d", id, lnd.ClassID) + } + found := false + for _, lip := range lnd.Priorities { + if lip.Name == ifName { + found = true + if lip.Priority != prio { + return fmt.Errorf("network priority for %s is not set correctly, expect: %d, actual: %d", ifName, prio, lip.Priority) + } + } + } + if !found { + return fmt.Errorf("network priority for %s is not set correctly", ifName) + } + + return nil + }) + + if err != nil { + util.Fatal(err) + } +} diff --git a/validation/linux_cgroups_pids.go b/validation/linux_cgroups_pids.go new file mode 100644 index 000000000..3e1fb780c --- /dev/null +++ b/validation/linux_cgroups_pids.go @@ -0,0 +1,33 @@ +package main + +import ( + "fmt" + + "github.com/opencontainers/runtime-tools/cgroups" + "github.com/opencontainers/runtime-tools/validation/util" +) + +func main() { + var limit int64 = 1000 + g := util.GetDefaultGenerator() + g.SetLinuxCgroupsPath(cgroups.AbsCgroupPath) + g.SetLinuxResourcesPidsLimit(limit) + err := util.RuntimeOutsideValidate(g, cgroups.AbsCgroupPath, func(pid int, path string) error { + cg, err := cgroups.FindCgroup() + if err != nil { + return err + } + lpd, err := cg.GetPidsData(pid, path) + if err != nil { + return err + } + if lpd.Limit != limit { + return fmt.Errorf("pids limit is not set correctly, expect: %d, actual: %d", limit, lpd.Limit) + } + return nil + }) + + if err != nil { + util.Fatal(err) + } +} diff --git a/validation/linux_cgroups_relative_blkio.go b/validation/linux_cgroups_relative_blkio.go new file mode 100644 index 000000000..4f2b56abd --- /dev/null +++ b/validation/linux_cgroups_relative_blkio.go @@ -0,0 +1,115 @@ +package main + +import ( + "fmt" + + "github.com/opencontainers/runtime-tools/cgroups" + "github.com/opencontainers/runtime-tools/validation/util" +) + +func main() { + var weight uint16 = 500 + var leafWeight uint16 = 300 + var major, minor int64 = 8, 0 + var rate uint64 = 102400 + g := util.GetDefaultGenerator() + g.SetLinuxCgroupsPath(cgroups.RelCgroupPath) + g.SetLinuxResourcesBlockIOWeight(weight) + g.SetLinuxResourcesBlockIOLeafWeight(leafWeight) + g.AddLinuxResourcesBlockIOWeightDevice(major, minor, weight) + g.AddLinuxResourcesBlockIOLeafWeightDevice(major, minor, leafWeight) + g.AddLinuxResourcesBlockIOThrottleReadBpsDevice(major, minor, rate) + g.AddLinuxResourcesBlockIOThrottleWriteBpsDevice(major, minor, rate) + g.AddLinuxResourcesBlockIOThrottleReadIOPSDevice(major, minor, rate) + g.AddLinuxResourcesBlockIOThrottleWriteIOPSDevice(major, minor, rate) + err := util.RuntimeOutsideValidate(g, cgroups.RelCgroupPath, func(pid int, path string) error { + cg, err := cgroups.FindCgroup() + if err != nil { + return err + } + lbd, err := cg.GetBlockIOData(pid, path) + if err != nil { + return err + } + if *lbd.Weight != weight { + return fmt.Errorf("blkio weight is not set correctly, expect: %d, actual: %d", weight, lbd.Weight) + } + if *lbd.LeafWeight != leafWeight { + return fmt.Errorf("blkio leafWeight is not set correctly, expect: %d, actual: %d", weight, lbd.LeafWeight) + } + + found := false + for _, wd := range lbd.WeightDevice { + if wd.Major == major && wd.Minor == minor { + found = true + if *wd.Weight != weight { + return fmt.Errorf("blkio weight for %d:%d is not set correctly, expect: %d, actual: %d", major, minor, weight, wd.Weight) + } + if *wd.LeafWeight != leafWeight { + return fmt.Errorf("blkio leafWeight for %d:%d is not set correctly, expect: %d, actual: %d", major, minor, leafWeight, wd.LeafWeight) + } + } + } + if !found { + return fmt.Errorf("blkio weightDevice for %d:%d is not set", major, minor) + } + + found = false + for _, trbd := range lbd.ThrottleReadBpsDevice { + if trbd.Major == major && trbd.Minor == minor { + found = true + if trbd.Rate != rate { + return fmt.Errorf("blkio read bps for %d:%d is not set correctly, expect: %d, actual: %d", major, minor, rate, trbd.Rate) + } + } + } + if !found { + return fmt.Errorf("blkio read bps for %d:%d is not set", major, minor) + } + + found = false + for _, twbd := range lbd.ThrottleWriteBpsDevice { + if twbd.Major == major && twbd.Minor == minor { + found = true + if twbd.Rate != rate { + return fmt.Errorf("blkio write bps for %d:%d is not set correctly, expect: %d, actual: %d", major, minor, rate, twbd.Rate) + } + } + } + if !found { + return fmt.Errorf("blkio write bps for %d:%d is not set", major, minor) + } + + found = false + for _, trid := range lbd.ThrottleReadIOPSDevice { + if trid.Major == major && trid.Minor == minor { + found = true + if trid.Rate != rate { + return fmt.Errorf("blkio read iops for %d:%d is not set correctly, expect: %d, actual: %d", major, minor, rate, trid.Rate) + } + } + } + if !found { + return fmt.Errorf("blkio read iops for %d:%d is not set", major, minor) + } + + found = false + for _, twid := range lbd.ThrottleWriteIOPSDevice { + if twid.Major == major && twid.Minor == minor { + found = true + if twid.Rate != rate { + return fmt.Errorf("blkio write iops for %d:%d is not set correctly, expect: %d, actual: %d", major, minor, rate, twid.Rate) + } + } + } + if !found { + return fmt.Errorf("blkio write iops for %d:%d is not set", major, minor) + } + + return nil + }) + + if err != nil { + util.Fatal(err) + } +} diff --git a/validation/linux_cgroups_relative_cpus.go b/validation/linux_cgroups_relative_cpus.go new file mode 100644 index 000000000..8102b77f9 --- /dev/null +++ b/validation/linux_cgroups_relative_cpus.go @@ -0,0 +1,52 @@ +package main + +import ( + "fmt" + + "github.com/opencontainers/runtime-tools/cgroups" + "github.com/opencontainers/runtime-tools/validation/util" +) + +func main() { + var shares uint64 = 1024 + var period uint64 = 100000 + var quota int64 = 50000 + var cpus, mems string = "0-1", "0" + g := util.GetDefaultGenerator() + g.SetLinuxCgroupsPath(cgroups.RelCgroupPath) + g.SetLinuxResourcesCPUShares(shares) + g.SetLinuxResourcesCPUQuota(quota) + g.SetLinuxResourcesCPUPeriod(period) + g.SetLinuxResourcesCPUCpus(cpus) + g.SetLinuxResourcesCPUMems(mems) + err := util.RuntimeOutsideValidate(g, cgroups.RelCgroupPath, func(pid int, path string) error { + cg, err := cgroups.FindCgroup() + if err != nil { + return err + } + lcd, err := cg.GetCPUData(pid, path) + if err != nil { + return err + } + if *lcd.Shares != shares { + return fmt.Errorf("cpus shares limit is not set correctly, expect: %d, actual: %d", shares, lcd.Shares) + } + if *lcd.Quota != quota { + return fmt.Errorf("cpus quota is not set correctly, expect: %d, actual: %d", quota, lcd.Quota) + } + if *lcd.Period != period { + return fmt.Errorf("cpus period is not set correctly, expect: %d, actual: %d", period, lcd.Period) + } + if lcd.Cpus != cpus { + return fmt.Errorf("cpus cpus is not set correctly, expect: %s, actual: %s", cpus, lcd.Cpus) + } + if lcd.Mems != mems { + return fmt.Errorf("cpus mems is not set correctly, expect: %s, actual: %s", mems, lcd.Mems) + } + return nil + }) + + if err != nil { + util.Fatal(err) + } +} diff --git a/validation/linux_cgroups_relative_hugetlb.go b/validation/linux_cgroups_relative_hugetlb.go new file mode 100644 index 000000000..5b888241f --- /dev/null +++ b/validation/linux_cgroups_relative_hugetlb.go @@ -0,0 +1,35 @@ +package main + +import ( + "fmt" + + "github.com/opencontainers/runtime-tools/cgroups" + "github.com/opencontainers/runtime-tools/validation/util" +) + +func main() { + page := "1GB" + var limit uint64 = 56892210544640 + g := util.GetDefaultGenerator() + g.SetLinuxCgroupsPath(cgroups.RelCgroupPath) + g.AddLinuxResourcesHugepageLimit(page, limit) + err := util.RuntimeOutsideValidate(g, cgroups.RelCgroupPath, func(pid int, path string) error { + cg, err := cgroups.FindCgroup() + if err != nil { + return err + } + lhd, err := cg.GetHugepageLimitData(pid, path) + if err != nil { + return err + } + for _, lhl := range lhd { + if lhl.Pagesize == page && lhl.Limit != limit { + return fmt.Errorf("hugepage %s limit is not set correctly, expect: %d, actual: %d", page, limit, lhl.Limit) + } + } + return nil + }) + if err != nil { + util.Fatal(err) + } +} diff --git a/validation/linux_cgroups_relative_memory.go b/validation/linux_cgroups_relative_memory.go new file mode 100644 index 000000000..578aa96dc --- /dev/null +++ b/validation/linux_cgroups_relative_memory.go @@ -0,0 +1,57 @@ +package main + +import ( + "fmt" + + "github.com/opencontainers/runtime-tools/cgroups" + "github.com/opencontainers/runtime-tools/validation/util" +) + +func main() { + var limit int64 = 50593792 + var swappiness uint64 = 50 + g := util.GetDefaultGenerator() + g.SetLinuxCgroupsPath(cgroups.RelCgroupPath) + g.SetLinuxResourcesMemoryLimit(limit) + g.SetLinuxResourcesMemoryReservation(limit) + g.SetLinuxResourcesMemorySwap(limit) + g.SetLinuxResourcesMemoryKernel(limit) + g.SetLinuxResourcesMemoryKernelTCP(limit) + g.SetLinuxResourcesMemorySwappiness(swappiness) + g.SetLinuxResourcesMemoryDisableOOMKiller(true) + err := util.RuntimeOutsideValidate(g, cgroups.RelCgroupPath, func(pid int, path string) error { + cg, err := cgroups.FindCgroup() + if err != nil { + return err + } + lm, err := cg.GetMemoryData(pid, path) + if err != nil { + return err + } + if limit != *lm.Limit { + return fmt.Errorf("memory limit is not set correctly, expect: %d, actual: %d", limit, *lm.Limit) + } + if limit != *lm.Reservation { + return fmt.Errorf("memory reservation is not set correctly, expect: %d, actual: %d", limit, *lm.Reservation) + } + if limit != *lm.Swap { + return fmt.Errorf("memory swap is not set correctly, expect: %d, actual: %d", limit, *lm.Reservation) + } + if limit != *lm.Kernel { + return fmt.Errorf("memory kernel is not set correctly, expect: %d, actual: %d", limit, *lm.Kernel) + } + if limit != *lm.KernelTCP { + return fmt.Errorf("memory kernelTCP is not set correctly, expect: %d, actual: %d", limit, *lm.Kernel) + } + if swappiness != *lm.Swappiness { + return fmt.Errorf("memory swappiness is not set correctly, expect: %d, actual: %d", swappiness, *lm.Swappiness) + } + if true != *lm.DisableOOMKiller { + return fmt.Errorf("memory oom is not set correctly, expect: %t, actual: %t", true, *lm.DisableOOMKiller) + } + return nil + }) + if err != nil { + util.Fatal(err) + } +} diff --git a/validation/linux_cgroups_relative_network.go b/validation/linux_cgroups_relative_network.go new file mode 100644 index 000000000..8d1985b3a --- /dev/null +++ b/validation/linux_cgroups_relative_network.go @@ -0,0 +1,47 @@ +package main + +import ( + "fmt" + + "github.com/opencontainers/runtime-tools/cgroups" + "github.com/opencontainers/runtime-tools/validation/util" +) + +func main() { + var id, prio uint32 = 255, 10 + ifName := "lo" + g := util.GetDefaultGenerator() + g.SetLinuxCgroupsPath(cgroups.RelCgroupPath) + g.SetLinuxResourcesNetworkClassID(id) + err := util.RuntimeOutsideValidate(g, cgroups.RelCgroupPath, func(pid int, path string) error { + cg, err := cgroups.FindCgroup() + if err != nil { + return err + } + lnd, err := cg.GetNetworkData(pid, path) + if err != nil { + return err + } + if *lnd.ClassID != id { + return fmt.Errorf("network ID is not set correctly, expect: %d, actual: %d", id, lnd.ClassID) + } + found := false + for _, lip := range lnd.Priorities { + if lip.Name == ifName { + found = true + if lip.Priority != prio { + return fmt.Errorf("network priority for %s is not set correctly, expect: %d, actual: %d", ifName, prio, lip.Priority) + } + } + } + if !found { + return fmt.Errorf("network priority for %s is not set correctly", ifName) + } + + return nil + }) + + if err != nil { + util.Fatal(err) + } +} diff --git a/validation/linux_cgroups_relative_pids.go b/validation/linux_cgroups_relative_pids.go new file mode 100644 index 000000000..f8ec4b145 --- /dev/null +++ b/validation/linux_cgroups_relative_pids.go @@ -0,0 +1,33 @@ +package main + +import ( + "fmt" + + "github.com/opencontainers/runtime-tools/cgroups" + "github.com/opencontainers/runtime-tools/validation/util" +) + +func main() { + var limit int64 = 1000 + g := util.GetDefaultGenerator() + g.SetLinuxCgroupsPath(cgroups.RelCgroupPath) + g.SetLinuxResourcesPidsLimit(limit) + err := util.RuntimeOutsideValidate(g, cgroups.RelCgroupPath, func(pid int, path string) error { + cg, err := cgroups.FindCgroup() + if err != nil { + return err + } + lpd, err := cg.GetPidsData(pid, path) + if err != nil { + return err + } + if lpd.Limit != limit { + return fmt.Errorf("pids limit is not set correctly, expect: %d, actual: %d", limit, lpd.Limit) + } + return nil + }) + + if err != nil { + util.Fatal(err) + } +} diff --git a/validation/util/test.go b/validation/util/test.go index 3b544197c..e18c3dbc5 100644 --- a/validation/util/test.go +++ b/validation/util/test.go @@ -24,6 +24,9 @@ var ( // but before creating the container. type PreFunc func(string) error +// AfterFunc validate container's outside environment after created +type AfterFunc func(int, string) error + func init() { runtimeInEnv := os.Getenv("RUNTIME") if runtimeInEnv != "" { @@ -138,3 +141,45 @@ func RuntimeInsideValidate(g *generate.Generator, f PreFunc) (err error) { os.Stdout.Write(stdout) return nil } + +// RuntimeOutsideValidate validate runtime outside a container. +func RuntimeOutsideValidate(g *generate.Generator, cgroupPath string, f AfterFunc) error { + bundleDir, err := PrepareBundle() + if err != nil { + return err + } + + r, err := NewRuntime(RuntimeCommand, bundleDir) + if err != nil { + os.RemoveAll(bundleDir) + return err + } + defer r.Clean(true, true) + err = r.SetConfig(g) + if err != nil { + return err + } + err = fileutils.CopyFile("runtimetest", filepath.Join(r.BundleDir, "runtimetest")) + if err != nil { + return err + } + + r.SetID(uuid.NewV4().String()) + stderr, err := r.Create() + if err != nil { + os.Stderr.WriteString("failed to start the container\n") + os.Stderr.Write(stderr) + return err + } + + if f != nil { + state, err := r.State() + if err != nil { + return err + } + if err := f(state.Pid, cgroupPath); err != nil { + return err + } + } + return nil +}