runtime: Fix hugepages support

Fixes: kata-containers#2353

Signed-off-by: Pradipta Banerjee <pradipta.banerjee@gmail.com>

virtcontainers/container.go

@@ -1358,7 +1358,7 @@ func (c *Container) hotplugDrive() error {
 			c.rootfsSuffix = ""
 		}
 		// If device mapper device, then fetch the full path of the device
-		devicePath, fsType, err = utils.GetDevicePathAndFsType(dev.mountPoint)
+		devicePath, fsType, _, err = utils.GetDevicePathAndFsTypeOptions(dev.mountPoint)
 		if err != nil {
 			return err
 		}

virtcontainers/device/config/pmem.go

@@ -75,7 +75,7 @@ func PmemDeviceInfo(source, destination string) (*DeviceInfo, error) {
 		return nil, fmt.Errorf("backing file %v has not PFN signature", device.HostPath)
 	}
 
-	_, fstype, err := utils.GetDevicePathAndFsType(source)
+	_, fstype, _, err := utils.GetDevicePathAndFsTypeOptions(source)
 	if err != nil {
 		pmemLog.WithError(err).WithField("mount-point", source).Warn("failed to get fstype: using ext4")
 		fstype = "ext4"

virtcontainers/kata_agent.go

@@ -18,6 +18,7 @@ import (
 	"syscall"
 	"time"
 
+	units "github.com/docker/go-units"
 	"github.com/gogo/protobuf/proto"
 	aTypes "github.com/kata-containers/agent/pkg/types"
 	kataclient "github.com/kata-containers/agent/protocols/client"
@@ -33,6 +34,7 @@ import (
 	"github.com/kata-containers/runtime/virtcontainers/pkg/uuid"
 	"github.com/kata-containers/runtime/virtcontainers/store"
 	"github.com/kata-containers/runtime/virtcontainers/types"
+	"github.com/kata-containers/runtime/virtcontainers/utils"
 	"github.com/opencontainers/runtime-spec/specs-go"
 	opentracing "github.com/opentracing/opentracing-go"
 	"github.com/sirupsen/logrus"
@@ -142,6 +144,17 @@ var kataHostSharedDir = func() string {
 	return defaultKataHostSharedDir
 }
 
+func getPagesizeFromOpt(fsOpts []string) string {
+	//example options array: "rw", "relatime", "seclabel", "pagesize=2M"
+
+	for _, opt := range fsOpts {
+		if strings.HasPrefix(opt, "pagesize=") {
+			return strings.TrimPrefix(opt, "pagesize=")
+		}
+	}
+	return ""
+}
+
 // Shared path handling:
 // 1. create two directories for each sandbox:
 // -. /run/kata-containers/shared/sandboxes/$sbx_id/mounts/, a directory to hold all host/guest shared mounts
@@ -1429,6 +1442,13 @@ func (k *kataAgent) createContainer(sandbox *Sandbox, c *Container) (p *Process,
 	epheStorages := k.handleEphemeralStorage(ociSpec.Mounts)
 	ctrStorages = append(ctrStorages, epheStorages...)
 
+	k.Logger().WithField("ociSpec Hugepage Resources", ociSpec.Linux.Resources.HugepageLimits).Debug("ociSpec HugepageLimit")
+	hugepages, err := k.handleHugepages(ociSpec.Mounts, ociSpec.Linux.Resources.HugepageLimits)
+	if err != nil {
+		return nil, err
+	}
+	ctrStorages = append(ctrStorages, hugepages...)
+
 	localStorages := k.handleLocalStorage(ociSpec.Mounts, sandbox.id, c.rootfsSuffix)
 	ctrStorages = append(ctrStorages, localStorages...)
 
@@ -1550,6 +1570,71 @@ func (k *kataAgent) handleEphemeralStorage(mounts []specs.Mount) []*grpc.Storage
 	return epheStorages
 }
 
+// handleHugePages handles hugepages storage by
+// creating a Storage from corresponding source of the mount point
+func (k *kataAgent) handleHugepages(mounts []specs.Mount, hugepageLimits []specs.LinuxHugepageLimit) ([]*grpc.Storage, error) {
+	//Map to hold the total memory of each type of hugepages
+	optionsMap := make(map[int64]string)
+
+	for _, hp := range hugepageLimits {
+		if hp.Limit != 0 {
+			k.Logger().WithFields(logrus.Fields{
+				"Pagesize": hp.Pagesize,
+				"Limit":    hp.Limit,
+			}).Info("hugepage request")
+			//example Pagesize 2MB, 1GB etc. The Limit are in Bytes
+			pageSize, err := units.RAMInBytes(hp.Pagesize)
+			if err != nil {
+				k.Logger().Error("Unable to convert pagesize to bytes")
+				return nil, err
+			}
+			totalHpSizeStr := strconv.FormatUint(hp.Limit, 10)
+			optionsMap[pageSize] = totalHpSizeStr
+		}
+	}
+
+	var hugepages []*grpc.Storage
+	for idx, mnt := range mounts {
+		if mnt.Type != KataLocalDevType {
+			continue
+		}
+		//HugePages mount Type is Local
+		if _, fsType, fsOptions, _ := utils.GetDevicePathAndFsTypeOptions(mnt.Source); fsType == "hugetlbfs" {
+			k.Logger().WithField("fsOptions", fsOptions).Debug("hugepage mount options")
+			//Find the pagesize from the mountpoint options
+			pagesizeOpt := getPagesizeFromOpt(fsOptions)
+			if pagesizeOpt == "" {
+				return nil, fmt.Errorf("No pagesize option found in filesystem mount options")
+			}
+			pageSize, err := units.RAMInBytes(pagesizeOpt)
+			if err != nil {
+				k.Logger().Error("Unable to convert pagesize from fs mount options to bytes")
+				return nil, err
+			}
+			//Create mount option string
+			options := fmt.Sprintf("pagesize=%s,size=%s", strconv.FormatInt(pageSize, 10), optionsMap[pageSize])
+			k.Logger().WithField("Hugepage options string", options).Debug("hugepage mount options")
+			// Set the mount source path to a path that resides inside the VM
+			mounts[idx].Source = filepath.Join(ephemeralPath(), filepath.Base(mnt.Source))
+			// Set the mount type to "bind"
+			mounts[idx].Type = "bind"
+
+			// Create a storage struct so that kata agent is able to create
+			// hugetlbfs backed volume inside the VM
+			hugepage := &grpc.Storage{
+				Driver:     KataEphemeralDevType,
+				Source:     "nodev",
+				Fstype:     "hugetlbfs",
+				MountPoint: mounts[idx].Source,
+				Options:    []string{options},
+			}
+			hugepages = append(hugepages, hugepage)
+		}
+
+	}
+	return hugepages, nil
+}
+
 // handleLocalStorage handles local storage within the VM
 // by creating a directory in the VM from the source of the mount point.
 func (k *kataAgent) handleLocalStorage(mounts []specs.Mount, sandboxID string, rootfsSuffix string) []*grpc.Storage {

virtcontainers/kata_agent_test.go

@@ -408,6 +408,61 @@ func TestHandleLocalStorage(t *testing.T) {
 	assert.Equal(t, localMountPoint, expected)
 }
 
+func TestHandleHugepages(t *testing.T) {
+	if os.Getuid() != 0 {
+		t.Skip("Test disabled as requires root user")
+	}
+
+	assert := assert.New(t)
+
+	dir, err := ioutil.TempDir("", "hugepages-test")
+	assert.Nil(err)
+	defer os.RemoveAll(dir)
+
+	k := kataAgent{}
+	var mounts []specs.Mount
+	var hugepageLimits []specs.LinuxHugepageLimit
+
+	hugepageDirs := [2]string{"hugepages-1Gi", "hugepages-2Mi"}
+	options := [2]string{"pagesize=1024M", "pagesize=2M"}
+
+	for i := 0; i < 2; i++ {
+		target := path.Join(dir, hugepageDirs[i])
+		err := os.MkdirAll(target, 0777)
+		assert.NoError(err, "Unable to create dir %s", target)
+
+		err = syscall.Mount("nodev", target, "hugetlbfs", uintptr(0), options[i])
+		assert.NoError(err, "Unable to mount %s", target)
+
+		defer syscall.Unmount(target, 0)
+		defer os.RemoveAll(target)
+		mount := specs.Mount{
+			Type:   KataLocalDevType,
+			Source: target,
+		}
+		mounts = append(mounts, mount)
+
+	}
+
+	hugepageLimits = []specs.LinuxHugepageLimit{
+		{
+			Pagesize: "1GB",
+			Limit:    322122547,
+		},
+		{
+			Pagesize: "2MB",
+			Limit:    134217728,
+		},
+	}
+
+	hugepages, err := k.handleHugepages(mounts, hugepageLimits)
+
+	assert.NoError(err, "Unable to handle hugepages %v", hugepageLimits)
+	assert.NotNil(hugepages)
+	assert.Equal(len(hugepages), 2)
+
+}
+
 func TestHandleDeviceBlockVolume(t *testing.T) {
 	k := kataAgent{}
 

virtcontainers/mount.go

@@ -400,7 +400,7 @@ func IsEphemeralStorage(path string) bool {
 		return false
 	}
 
-	if _, fsType, _ := utils.GetDevicePathAndFsType(path); fsType == "tmpfs" {
+	if _, fsType, _, _ := utils.GetDevicePathAndFsTypeOptions(path); fsType == "tmpfs" {
 		return true
 	}
 
@@ -415,7 +415,7 @@ func Isk8sHostEmptyDir(path string) bool {
 		return false
 	}
 
-	if _, fsType, _ := utils.GetDevicePathAndFsType(path); fsType != "tmpfs" {
+	if _, fsType, _, _ := utils.GetDevicePathAndFsTypeOptions(path); fsType != "tmpfs" {
 		return true
 	}
 	return false

virtcontainers/utils/utils_linux.go

@@ -99,11 +99,12 @@ const (
 	procDeviceIndex = iota
 	procPathIndex
 	procTypeIndex
+	procOptionIndex
 )
 
-// GetDevicePathAndFsType gets the device for the mount point and the file system type
-// of the mount.
-func GetDevicePathAndFsType(mountPoint string) (devicePath, fsType string, err error) {
+// GetDevicePathAndFsTypeOptions gets the device for the mount point, the file system type
+// and mount options
+func GetDevicePathAndFsTypeOptions(mountPoint string) (devicePath, fsType string, fsOptions []string, err error) {
 	if mountPoint == "" {
 		err = fmt.Errorf("Mount point cannot be empty")
 		return
@@ -137,6 +138,7 @@ func GetDevicePathAndFsType(mountPoint string) (devicePath, fsType string, err e
 		if mountPoint == fields[procPathIndex] {
 			devicePath = fields[procDeviceIndex]
 			fsType = fields[procTypeIndex]
+			fsOptions = strings.Split(fields[procOptionIndex], ",")
 			return
 		}
 	}

virtcontainers/utils/utils_linux_test.go

@@ -6,7 +6,10 @@
 package utils
 
 import (
+	"bytes"
 	"errors"
+	"os/exec"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -36,16 +39,27 @@ func TestFindContextID(t *testing.T) {
 
 func TestGetDevicePathAndFsTypeEmptyMount(t *testing.T) {
 	assert := assert.New(t)
-	_, _, err := GetDevicePathAndFsType("")
+	_, _, _, err := GetDevicePathAndFsTypeOptions("")
 	assert.Error(err)
 }
 
 func TestGetDevicePathAndFsTypeSuccessful(t *testing.T) {
 	assert := assert.New(t)
 
-	path, fstype, err := GetDevicePathAndFsType("/proc")
+	cmdStr := "grep ^proc  /proc/mounts"
+	cmd := exec.Command("sh", "-c", cmdStr)
+	output, err := cmd.Output()
+	assert.NoError(err)
+
+	data := bytes.Split(output, []byte(" "))
+	fstypeOut := string(data[2])
+	optsOut := strings.Split(string(data[3]), ",")
+
+	path, fstype, fsOptions, err := GetDevicePathAndFsTypeOptions("/proc")
 	assert.NoError(err)
 
 	assert.Equal(path, "proc")
 	assert.Equal(fstype, "proc")
+	assert.Equal(fstype, fstypeOut)
+	assert.Equal(fsOptions, optsOut)
 }