runtime: Fix hugepages support

Fixes: kata-containers#2353

Signed-off-by: Pradipta Banerjee <pradipta.banerjee@gmail.com>

virtcontainers/container.go

@@ -1358,7 +1358,7 @@ func (c *Container) hotplugDrive() error {
 			c.rootfsSuffix = ""
 		}
 		// If device mapper device, then fetch the full path of the device
-		devicePath, fsType, err = utils.GetDevicePathAndFsType(dev.mountPoint)
+		devicePath, fsType, _, err = utils.GetDevicePathAndFsTypeOptions(dev.mountPoint)
 		if err != nil {
 			return err
 		}

virtcontainers/device/config/pmem.go

@@ -75,7 +75,7 @@ func PmemDeviceInfo(source, destination string) (*DeviceInfo, error) {
 		return nil, fmt.Errorf("backing file %v has not PFN signature", device.HostPath)
 	}
 
-	_, fstype, err := utils.GetDevicePathAndFsType(source)
+	_, fstype, _, err := utils.GetDevicePathAndFsTypeOptions(source)
 	if err != nil {
 		pmemLog.WithError(err).WithField("mount-point", source).Warn("failed to get fstype: using ext4")
 		fstype = "ext4"

virtcontainers/kata_agent.go

@@ -18,6 +18,7 @@ import (
 	"syscall"
 	"time"
 
+	units "github.com/docker/go-units"
 	"github.com/gogo/protobuf/proto"
 	aTypes "github.com/kata-containers/agent/pkg/types"
 	kataclient "github.com/kata-containers/agent/protocols/client"
@@ -33,6 +34,7 @@ import (
 	"github.com/kata-containers/runtime/virtcontainers/pkg/uuid"
 	"github.com/kata-containers/runtime/virtcontainers/store"
 	"github.com/kata-containers/runtime/virtcontainers/types"
+	"github.com/kata-containers/runtime/virtcontainers/utils"
 	"github.com/opencontainers/runtime-spec/specs-go"
 	opentracing "github.com/opentracing/opentracing-go"
 	"github.com/sirupsen/logrus"
@@ -142,6 +144,17 @@ var kataHostSharedDir = func() string {
 	return defaultKataHostSharedDir
 }
 
+func getPagesizeFromOpt(fsOpts []string) string {
+	//example options array: "rw", "relatime", "seclabel", "pagesize=2M"
+
+	for _, opt := range fsOpts {
+		if strings.Contains(opt, "pagesize") {
+			return strings.TrimPrefix(opt, "pagesize=")
+		}
+	}
+	return ""
+}
+
 // Shared path handling:
 // 1. create two directories for each sandbox:
 // -. /run/kata-containers/shared/sandboxes/$sbx_id/mounts/, a directory to hold all host/guest shared mounts
@@ -1429,6 +1442,13 @@ func (k *kataAgent) createContainer(sandbox *Sandbox, c *Container) (p *Process,
 	epheStorages := k.handleEphemeralStorage(ociSpec.Mounts)
 	ctrStorages = append(ctrStorages, epheStorages...)
 
+	k.Logger().WithField("ociSpec Hugepage Resources", ociSpec.Linux.Resources.HugepageLimits).Debug("ociSpec HugepageLimit")
+	hugepages, err := k.handleHugepages(ociSpec.Mounts, ociSpec.Linux.Resources.HugepageLimits)
+	if err != nil {
+		return nil, err
+	}
+	ctrStorages = append(ctrStorages, hugepages...)
+
 	localStorages := k.handleLocalStorage(ociSpec.Mounts, sandbox.id, c.rootfsSuffix)
 	ctrStorages = append(ctrStorages, localStorages...)
 
@@ -1550,6 +1570,72 @@ func (k *kataAgent) handleEphemeralStorage(mounts []specs.Mount) []*grpc.Storage
 	return epheStorages
 }
 
+// handleHugePages handles hugepages storage by
+// creating a Storage from corresponding source of the mount point
+func (k *kataAgent) handleHugepages(mounts []specs.Mount, hugepageLimits []specs.LinuxHugepageLimit) ([]*grpc.Storage, error) {
+	//Map to hold the total memory of each type of hugepages
+	optionsMap := make(map[int64]string)
+
+	for _, hp := range hugepageLimits {
+		if hp.Limit != 0 {
+			k.Logger().WithFields(logrus.Fields{
+				"Pagesize": hp.Pagesize,
+				"Limit":    hp.Limit,
+			}).Info("hugepage request")
+			//example Pagesize 2MB, 1GB etc. The Limit are in Bytes
+			pageSize, err := units.RAMInBytes(hp.Pagesize)
+			if err != nil {
+				k.Logger().Error("Unable to convert pagesize to bytes")
+				return nil, err
+			}
+			totalHpSizeStr := strconv.FormatUint(hp.Limit, 10)
+			optionsMap[pageSize] = totalHpSizeStr
+		}
+	}
+
+	var hugepages []*grpc.Storage
+	for idx, mnt := range mounts {
+		if mnt.Type != KataLocalDevType {
+			continue
+		}
+		//HugePages mount Type is Local
+		if _, fsType, fsOptions, _ := utils.GetDevicePathAndFsTypeOptions(mnt.Source); fsType == "hugetlbfs" {
+			k.Logger().WithField("fsOptions", fsOptions).Debug("hugepage mount options")
+			//Find the pagesize from the mountpoint options
+			pagesizeOpt := getPagesizeFromOpt(fsOptions)
+			if pagesizeOpt == "" {
+				return nil, fmt.Errorf("No pagesize option found in filesystem mount options")
+			}
+			pageSize, err := units.RAMInBytes(pagesizeOpt)
+			if err != nil {
+				k.Logger().Error("Unable to convert pagesize from fs mount options to bytes")
+				return nil, err
+			}
+			pageSizeStr := strconv.FormatInt(pageSize, 10)
+			//Create mount option string
+			options := "pagesize=" + pageSizeStr + "," + "size=" + optionsMap[pageSize]
+			k.Logger().WithField("Hugepage options string", options).Debug("hugepage mount options")
+			// Set the mount source path to a path that resides inside the VM
+			mounts[idx].Source = filepath.Join(ephemeralPath(), filepath.Base(mnt.Source))
+			// Set the mount type to "bind"
+			mounts[idx].Type = "bind"
+
+			// Create a storage struct so that kata agent is able to create
+			// hugetlbfs backed volume inside the VM
+			hugepage := &grpc.Storage{
+				Driver:     KataEphemeralDevType,
+				Source:     "nodev",
+				Fstype:     "hugetlbfs",
+				MountPoint: mounts[idx].Source,
+				Options:    []string{options},
+			}
+			hugepages = append(hugepages, hugepage)
+		}
+
+	}
+	return hugepages, nil
+}
+
 // handleLocalStorage handles local storage within the VM
 // by creating a directory in the VM from the source of the mount point.
 func (k *kataAgent) handleLocalStorage(mounts []specs.Mount, sandboxID string, rootfsSuffix string) []*grpc.Storage {

virtcontainers/kata_agent_test.go

@@ -408,6 +408,62 @@ func TestHandleLocalStorage(t *testing.T) {
 	assert.Equal(t, localMountPoint, expected)
 }
 
+func TestHandleHugepages(t *testing.T) {
+	if os.Getuid() != 0 {
+		t.Skip("Test disabled as requires root user")
+	}
+
+	dir, err := ioutil.TempDir("", "hugepages-test")
+	assert.Nil(t, err)
+	defer os.RemoveAll(dir)
+
+	k := kataAgent{}
+	var mounts []specs.Mount
+	var hugepageLimits []specs.LinuxHugepageLimit
+
+	hugepageDirs := [2]string{"hugepages-1Gi", "hugepages-2Mi"}
+	options := [2]string{"pagesize=1024M", "pagesize=2M"}
+
+	for i := 0; i < 2; i++ {
+		target := path.Join(dir, hugepageDirs[i])
+		err := os.MkdirAll(target, 0777)
+		if err = syscall.Mount("nodev", target, "hugetlbfs", uintptr(0), options[i]); err != nil {
+			fmt.Printf("Unable to mount %s: %v\n", target, err)
+		}
+		assert.Nil(t, err)
+		defer syscall.Unmount(target, 0)
+		defer os.RemoveAll(target)
+		mount := specs.Mount{
+			Type:   KataLocalDevType,
+			Source: target,
+		}
+		mounts = append(mounts, mount)
+
+	}
+	fmt.Printf("mounts: %v\n", mounts)
+
+	hugepageLimits = []specs.LinuxHugepageLimit{
+		{
+			Pagesize: "1GB",
+			Limit:    322122547,
+		},
+		{
+			Pagesize: "2MB",
+			Limit:    134217728,
+		},
+	}
+	fmt.Printf("hugepageLimits: %v\n", hugepageLimits)
+
+	hugepages, err := k.handleHugepages(mounts, hugepageLimits)
+
+	fmt.Printf("%v\n", hugepages)
+
+	assert.NotNil(t, hugepages)
+	assert.Equal(t, len(hugepages), 2)
+	assert.NoError(t, err)
+
+}
+
 func TestHandleDeviceBlockVolume(t *testing.T) {
 	k := kataAgent{}
 

virtcontainers/mount.go

@@ -400,7 +400,7 @@ func IsEphemeralStorage(path string) bool {
 		return false
 	}
 
-	if _, fsType, _ := utils.GetDevicePathAndFsType(path); fsType == "tmpfs" {
+	if _, fsType, _, _ := utils.GetDevicePathAndFsTypeOptions(path); fsType == "tmpfs" {
 		return true
 	}
 
@@ -415,7 +415,7 @@ func Isk8sHostEmptyDir(path string) bool {
 		return false
 	}
 
-	if _, fsType, _ := utils.GetDevicePathAndFsType(path); fsType != "tmpfs" {
+	if _, fsType, _, _ := utils.GetDevicePathAndFsTypeOptions(path); fsType != "tmpfs" {
 		return true
 	}
 	return false

virtcontainers/sandbox.go

@@ -1873,8 +1873,9 @@ func (s *Sandbox) updateResources() error {
 	sandboxVCPUs += s.hypervisor.hypervisorConfig().NumVCPUs
 
 	sandboxMemoryByte := s.calculateSandboxMemory()
+
 	// Add default / rsvd memory for sandbox.
-	sandboxMemoryByte += int64(s.hypervisor.hypervisorConfig().MemorySize) << utils.MibToBytesShift
+	sandboxMemoryByte += uint64(s.hypervisor.hypervisorConfig().MemorySize) << utils.MibToBytesShift
 
 	// Update VCPUs
 	s.Logger().WithField("cpus-sandbox", sandboxVCPUs).Debugf("Request to hypervisor to update vCPUs")
@@ -1894,7 +1895,9 @@ func (s *Sandbox) updateResources() error {
 
 	// Update Memory
 	s.Logger().WithField("memory-sandbox-size-byte", sandboxMemoryByte).Debugf("Request to hypervisor to update memory")
-	newMemory, updatedMemoryDevice, err := s.hypervisor.resizeMemory(uint32(sandboxMemoryByte>>utils.MibToBytesShift), s.state.GuestMemoryBlockSizeMB, s.state.GuestMemoryHotplugProbe)
+	newMemoryMB := uint32(sandboxMemoryByte >> utils.MibToBytesShift)
+
+	newMemory, updatedMemoryDevice, err := s.hypervisor.resizeMemory(newMemoryMB, s.state.GuestMemoryBlockSizeMB, s.state.GuestMemoryHotplugProbe)
 	if err != nil {
 		return err
 	}
@@ -1912,19 +1915,27 @@ func (s *Sandbox) updateResources() error {
 	return nil
 }
 
-func (s *Sandbox) calculateSandboxMemory() int64 {
-	memorySandbox := int64(0)
+func (s *Sandbox) calculateSandboxMemory() uint64 {
+	memorySandbox := uint64(0)
 	for _, c := range s.config.Containers {
 		// Do not hot add again non-running containers resources
 		if cont, ok := s.containers[c.ID]; ok && cont.state.State == types.StateStopped {
 			s.Logger().WithField("container-id", c.ID).Debug("Do not taking into account memory resources of not running containers")
 			continue
 		}
 
-		if m := c.Resources.Memory; m != nil && m.Limit != nil {
-			memorySandbox += *m.Limit
+		if m := c.Resources.Memory; m != nil && m.Limit != nil && *m.Limit > 0 {
+			memorySandbox += uint64(*m.Limit)
+			s.Logger().WithField("memory limit", memorySandbox).Info("Memory Sandbox + Memory Limit ")
+		}
+
+		//Add hugepages memory
+		//HugepageLimit is uint64 - https://github.com/opencontainers/runtime-spec/blob/master/specs-go/config.go#L242
+		for _, l := range c.Resources.HugepageLimits {
+			memorySandbox += uint64(l.Limit)
 		}
 	}
+
 	return memorySandbox
 }
 

virtcontainers/sandbox_test.go

@@ -149,13 +149,14 @@ func TestCalculateSandboxMem(t *testing.T) {
 	sandbox.config = &SandboxConfig{}
 	unconstrained := newTestContainerConfigNoop("cont-00001")
 	constrained := newTestContainerConfigNoop("cont-00001")
-	limit := int64(4000)
-	constrained.Resources.Memory = &specs.LinuxMemory{Limit: &limit}
+	mlimit := int64(4000)
+	limit := uint64(4000)
+	constrained.Resources.Memory = &specs.LinuxMemory{Limit: &mlimit}
 
 	tests := []struct {
 		name       string
 		containers []ContainerConfig
-		want       int64
+		want       uint64
 	}{
 		{"1-unconstrained", []ContainerConfig{unconstrained}, 0},
 		{"2-unconstrained", []ContainerConfig{unconstrained, unconstrained}, 0},
@@ -173,6 +174,45 @@ func TestCalculateSandboxMem(t *testing.T) {
 	}
 }
 
+func TestSandboxHugepageLimit(t *testing.T) {
+	contConfig1 := newTestContainerConfigNoop("cont-00001")
+	contConfig2 := newTestContainerConfigNoop("cont-00002")
+	limit := int64(4000)
+	contConfig1.Resources.Memory = &specs.LinuxMemory{Limit: &limit}
+	contConfig2.Resources.Memory = &specs.LinuxMemory{Limit: &limit}
+	hConfig := newHypervisorConfig(nil, nil)
+
+	defer cleanUp()
+	// create a sandbox
+	s, err := testCreateSandbox(t,
+		testSandboxID,
+		MockHypervisor,
+		hConfig,
+		NoopAgentType,
+		NetworkConfig{},
+		[]ContainerConfig{contConfig1, contConfig2},
+		nil)
+
+	assert.NoError(t, err)
+
+	hugepageLimits := []specs.LinuxHugepageLimit{
+		{
+			Pagesize: "1GB",
+			Limit:    322122547,
+		},
+		{
+			Pagesize: "2MB",
+			Limit:    134217728,
+		},
+	}
+
+	for i := range s.config.Containers {
+		s.config.Containers[i].Resources.HugepageLimits = hugepageLimits
+	}
+	err = s.updateResources()
+	assert.NoError(t, err)
+}
+
 func TestCreateSandboxEmptyID(t *testing.T) {
 	hConfig := newHypervisorConfig(nil, nil)
 	_, err := testCreateSandbox(t, "", MockHypervisor, hConfig, NoopAgentType, NetworkConfig{}, nil, nil)

virtcontainers/utils/utils_linux.go

@@ -99,11 +99,12 @@ const (
 	procDeviceIndex = iota
 	procPathIndex
 	procTypeIndex
+	procOptionIndex
 )
 
-// GetDevicePathAndFsType gets the device for the mount point and the file system type
-// of the mount.
-func GetDevicePathAndFsType(mountPoint string) (devicePath, fsType string, err error) {
+// GetDevicePathAndFsTypeOptions gets the device for the mount point, the file system type
+// and mount options
+func GetDevicePathAndFsTypeOptions(mountPoint string) (devicePath, fsType string, fsOptions []string, err error) {
 	if mountPoint == "" {
 		err = fmt.Errorf("Mount point cannot be empty")
 		return
@@ -137,6 +138,7 @@ func GetDevicePathAndFsType(mountPoint string) (devicePath, fsType string, err e
 		if mountPoint == fields[procPathIndex] {
 			devicePath = fields[procDeviceIndex]
 			fsType = fields[procTypeIndex]
+			fsOptions = strings.Split(fields[procOptionIndex], ",")
 			return
 		}
 	}

virtcontainers/utils/utils_linux_test.go

@@ -36,16 +36,18 @@ func TestFindContextID(t *testing.T) {
 
 func TestGetDevicePathAndFsTypeEmptyMount(t *testing.T) {
 	assert := assert.New(t)
-	_, _, err := GetDevicePathAndFsType("")
+	_, _, _, err := GetDevicePathAndFsTypeOptions("")
 	assert.Error(err)
 }
 
 func TestGetDevicePathAndFsTypeSuccessful(t *testing.T) {
 	assert := assert.New(t)
+	opts := []string{"rw", "nosuid", "nodev", "noexec", "relatime"}
 
-	path, fstype, err := GetDevicePathAndFsType("/proc")
+	path, fstype, fsOptions, err := GetDevicePathAndFsTypeOptions("/proc")
 	assert.NoError(err)
 
 	assert.Equal(path, "proc")
 	assert.Equal(fstype, "proc")
+	assert.True(DeepCompare(fsOptions, opts))
 }