diff --git a/virtcontainers/container.go b/virtcontainers/container.go index 37117d7e38..57a67fd485 100644 --- a/virtcontainers/container.go +++ b/virtcontainers/container.go @@ -1358,7 +1358,7 @@ func (c *Container) hotplugDrive() error { c.rootfsSuffix = "" } // If device mapper device, then fetch the full path of the device - devicePath, fsType, err = utils.GetDevicePathAndFsType(dev.mountPoint) + devicePath, fsType, _, err = utils.GetDevicePathAndFsTypeOptions(dev.mountPoint) if err != nil { return err } diff --git a/virtcontainers/device/config/pmem.go b/virtcontainers/device/config/pmem.go index 1b4b80f2b5..c4f723cf7f 100644 --- a/virtcontainers/device/config/pmem.go +++ b/virtcontainers/device/config/pmem.go @@ -75,7 +75,7 @@ func PmemDeviceInfo(source, destination string) (*DeviceInfo, error) { return nil, fmt.Errorf("backing file %v has not PFN signature", device.HostPath) } - _, fstype, err := utils.GetDevicePathAndFsType(source) + _, fstype, _, err := utils.GetDevicePathAndFsTypeOptions(source) if err != nil { pmemLog.WithError(err).WithField("mount-point", source).Warn("failed to get fstype: using ext4") fstype = "ext4" diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go index 934b1bee82..78a0435734 100644 --- a/virtcontainers/kata_agent.go +++ b/virtcontainers/kata_agent.go @@ -18,6 +18,7 @@ import ( "syscall" "time" + units "github.com/docker/go-units" "github.com/gogo/protobuf/proto" aTypes "github.com/kata-containers/agent/pkg/types" kataclient "github.com/kata-containers/agent/protocols/client" @@ -33,6 +34,7 @@ import ( "github.com/kata-containers/runtime/virtcontainers/pkg/uuid" "github.com/kata-containers/runtime/virtcontainers/store" "github.com/kata-containers/runtime/virtcontainers/types" + "github.com/kata-containers/runtime/virtcontainers/utils" "github.com/opencontainers/runtime-spec/specs-go" opentracing "github.com/opentracing/opentracing-go" "github.com/sirupsen/logrus" @@ -142,6 +144,17 @@ var kataHostSharedDir = func() string { return defaultKataHostSharedDir } +func getPagesizeFromOpt(fsOpts []string) string { + //example options array: "rw", "relatime", "seclabel", "pagesize=2M" + + for _, opt := range fsOpts { + if strings.HasPrefix(opt, "pagesize=") { + return strings.TrimPrefix(opt, "pagesize=") + } + } + return "" +} + // Shared path handling: // 1. create two directories for each sandbox: // -. /run/kata-containers/shared/sandboxes/$sbx_id/mounts/, a directory to hold all host/guest shared mounts @@ -1429,6 +1442,13 @@ func (k *kataAgent) createContainer(sandbox *Sandbox, c *Container) (p *Process, epheStorages := k.handleEphemeralStorage(ociSpec.Mounts) ctrStorages = append(ctrStorages, epheStorages...) + k.Logger().WithField("ociSpec Hugepage Resources", ociSpec.Linux.Resources.HugepageLimits).Debug("ociSpec HugepageLimit") + hugepages, err := k.handleHugepages(ociSpec.Mounts, ociSpec.Linux.Resources.HugepageLimits) + if err != nil { + return nil, err + } + ctrStorages = append(ctrStorages, hugepages...) + localStorages := k.handleLocalStorage(ociSpec.Mounts, sandbox.id, c.rootfsSuffix) ctrStorages = append(ctrStorages, localStorages...) @@ -1550,6 +1570,71 @@ func (k *kataAgent) handleEphemeralStorage(mounts []specs.Mount) []*grpc.Storage return epheStorages } +// handleHugePages handles hugepages storage by +// creating a Storage from corresponding source of the mount point +func (k *kataAgent) handleHugepages(mounts []specs.Mount, hugepageLimits []specs.LinuxHugepageLimit) ([]*grpc.Storage, error) { + //Map to hold the total memory of each type of hugepages + optionsMap := make(map[int64]string) + + for _, hp := range hugepageLimits { + if hp.Limit != 0 { + k.Logger().WithFields(logrus.Fields{ + "Pagesize": hp.Pagesize, + "Limit": hp.Limit, + }).Info("hugepage request") + //example Pagesize 2MB, 1GB etc. The Limit are in Bytes + pageSize, err := units.RAMInBytes(hp.Pagesize) + if err != nil { + k.Logger().Error("Unable to convert pagesize to bytes") + return nil, err + } + totalHpSizeStr := strconv.FormatUint(hp.Limit, 10) + optionsMap[pageSize] = totalHpSizeStr + } + } + + var hugepages []*grpc.Storage + for idx, mnt := range mounts { + if mnt.Type != KataLocalDevType { + continue + } + //HugePages mount Type is Local + if _, fsType, fsOptions, _ := utils.GetDevicePathAndFsTypeOptions(mnt.Source); fsType == "hugetlbfs" { + k.Logger().WithField("fsOptions", fsOptions).Debug("hugepage mount options") + //Find the pagesize from the mountpoint options + pagesizeOpt := getPagesizeFromOpt(fsOptions) + if pagesizeOpt == "" { + return nil, fmt.Errorf("No pagesize option found in filesystem mount options") + } + pageSize, err := units.RAMInBytes(pagesizeOpt) + if err != nil { + k.Logger().Error("Unable to convert pagesize from fs mount options to bytes") + return nil, err + } + //Create mount option string + options := fmt.Sprintf("pagesize=%s,size=%s", strconv.FormatInt(pageSize, 10), optionsMap[pageSize]) + k.Logger().WithField("Hugepage options string", options).Debug("hugepage mount options") + // Set the mount source path to a path that resides inside the VM + mounts[idx].Source = filepath.Join(ephemeralPath(), filepath.Base(mnt.Source)) + // Set the mount type to "bind" + mounts[idx].Type = "bind" + + // Create a storage struct so that kata agent is able to create + // hugetlbfs backed volume inside the VM + hugepage := &grpc.Storage{ + Driver: KataEphemeralDevType, + Source: "nodev", + Fstype: "hugetlbfs", + MountPoint: mounts[idx].Source, + Options: []string{options}, + } + hugepages = append(hugepages, hugepage) + } + + } + return hugepages, nil +} + // handleLocalStorage handles local storage within the VM // by creating a directory in the VM from the source of the mount point. func (k *kataAgent) handleLocalStorage(mounts []specs.Mount, sandboxID string, rootfsSuffix string) []*grpc.Storage { diff --git a/virtcontainers/kata_agent_test.go b/virtcontainers/kata_agent_test.go index 0041184fa0..32c4328a84 100644 --- a/virtcontainers/kata_agent_test.go +++ b/virtcontainers/kata_agent_test.go @@ -408,6 +408,61 @@ func TestHandleLocalStorage(t *testing.T) { assert.Equal(t, localMountPoint, expected) } +func TestHandleHugepages(t *testing.T) { + if os.Getuid() != 0 { + t.Skip("Test disabled as requires root user") + } + + assert := assert.New(t) + + dir, err := ioutil.TempDir("", "hugepages-test") + assert.Nil(err) + defer os.RemoveAll(dir) + + k := kataAgent{} + var mounts []specs.Mount + var hugepageLimits []specs.LinuxHugepageLimit + + hugepageDirs := [2]string{"hugepages-1Gi", "hugepages-2Mi"} + options := [2]string{"pagesize=1024M", "pagesize=2M"} + + for i := 0; i < 2; i++ { + target := path.Join(dir, hugepageDirs[i]) + err := os.MkdirAll(target, 0777) + assert.NoError(err, "Unable to create dir %s", target) + + err = syscall.Mount("nodev", target, "hugetlbfs", uintptr(0), options[i]) + assert.NoError(err, "Unable to mount %s", target) + + defer syscall.Unmount(target, 0) + defer os.RemoveAll(target) + mount := specs.Mount{ + Type: KataLocalDevType, + Source: target, + } + mounts = append(mounts, mount) + + } + + hugepageLimits = []specs.LinuxHugepageLimit{ + { + Pagesize: "1GB", + Limit: 322122547, + }, + { + Pagesize: "2MB", + Limit: 134217728, + }, + } + + hugepages, err := k.handleHugepages(mounts, hugepageLimits) + + assert.NoError(err, "Unable to handle hugepages %v", hugepageLimits) + assert.NotNil(hugepages) + assert.Equal(len(hugepages), 2) + +} + func TestHandleDeviceBlockVolume(t *testing.T) { k := kataAgent{} diff --git a/virtcontainers/mount.go b/virtcontainers/mount.go index 2f9846e809..f19690992d 100644 --- a/virtcontainers/mount.go +++ b/virtcontainers/mount.go @@ -400,7 +400,7 @@ func IsEphemeralStorage(path string) bool { return false } - if _, fsType, _ := utils.GetDevicePathAndFsType(path); fsType == "tmpfs" { + if _, fsType, _, _ := utils.GetDevicePathAndFsTypeOptions(path); fsType == "tmpfs" { return true } @@ -415,7 +415,7 @@ func Isk8sHostEmptyDir(path string) bool { return false } - if _, fsType, _ := utils.GetDevicePathAndFsType(path); fsType != "tmpfs" { + if _, fsType, _, _ := utils.GetDevicePathAndFsTypeOptions(path); fsType != "tmpfs" { return true } return false diff --git a/virtcontainers/utils/utils_linux.go b/virtcontainers/utils/utils_linux.go index ad870d63ee..caacf5fed3 100644 --- a/virtcontainers/utils/utils_linux.go +++ b/virtcontainers/utils/utils_linux.go @@ -99,11 +99,12 @@ const ( procDeviceIndex = iota procPathIndex procTypeIndex + procOptionIndex ) -// GetDevicePathAndFsType gets the device for the mount point and the file system type -// of the mount. -func GetDevicePathAndFsType(mountPoint string) (devicePath, fsType string, err error) { +// GetDevicePathAndFsTypeOptions gets the device for the mount point, the file system type +// and mount options +func GetDevicePathAndFsTypeOptions(mountPoint string) (devicePath, fsType string, fsOptions []string, err error) { if mountPoint == "" { err = fmt.Errorf("Mount point cannot be empty") return @@ -137,6 +138,7 @@ func GetDevicePathAndFsType(mountPoint string) (devicePath, fsType string, err e if mountPoint == fields[procPathIndex] { devicePath = fields[procDeviceIndex] fsType = fields[procTypeIndex] + fsOptions = strings.Split(fields[procOptionIndex], ",") return } } diff --git a/virtcontainers/utils/utils_linux_test.go b/virtcontainers/utils/utils_linux_test.go index 4554fa935d..98ca2359cb 100644 --- a/virtcontainers/utils/utils_linux_test.go +++ b/virtcontainers/utils/utils_linux_test.go @@ -6,7 +6,10 @@ package utils import ( + "bytes" "errors" + "os/exec" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -36,16 +39,27 @@ func TestFindContextID(t *testing.T) { func TestGetDevicePathAndFsTypeEmptyMount(t *testing.T) { assert := assert.New(t) - _, _, err := GetDevicePathAndFsType("") + _, _, _, err := GetDevicePathAndFsTypeOptions("") assert.Error(err) } func TestGetDevicePathAndFsTypeSuccessful(t *testing.T) { assert := assert.New(t) - path, fstype, err := GetDevicePathAndFsType("/proc") + cmdStr := "grep ^proc /proc/mounts" + cmd := exec.Command("sh", "-c", cmdStr) + output, err := cmd.Output() + assert.NoError(err) + + data := bytes.Split(output, []byte(" ")) + fstypeOut := string(data[2]) + optsOut := strings.Split(string(data[3]), ",") + + path, fstype, fsOptions, err := GetDevicePathAndFsTypeOptions("/proc") assert.NoError(err) assert.Equal(path, "proc") assert.Equal(fstype, "proc") + assert.Equal(fstype, fstypeOut) + assert.Equal(fsOptions, optsOut) }