Skip to content

Commit

Permalink
runtime: Fix hugepages support
Browse files Browse the repository at this point in the history
Fixes: kata-containers#2353

Signed-off-by: Pradipta Banerjee <pradipta.banerjee@gmail.com>
  • Loading branch information
bpradipt committed Jan 8, 2021
1 parent 2a98f43 commit 054ca93
Show file tree
Hide file tree
Showing 9 changed files with 240 additions and 13 deletions.
2 changes: 1 addition & 1 deletion virtcontainers/container.go
Original file line number Diff line number Diff line change
Expand Up @@ -1358,7 +1358,7 @@ func (c *Container) hotplugDrive() error {
c.rootfsSuffix = ""
}
// If device mapper device, then fetch the full path of the device
devicePath, fsType, err = utils.GetDevicePathAndFsType(dev.mountPoint)
devicePath, fsType, _, err = utils.GetDevicePathAndFsTypeOptions(dev.mountPoint)
if err != nil {
return err
}
Expand Down
2 changes: 1 addition & 1 deletion virtcontainers/device/config/pmem.go
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ func PmemDeviceInfo(source, destination string) (*DeviceInfo, error) {
return nil, fmt.Errorf("backing file %v has not PFN signature", device.HostPath)
}

_, fstype, err := utils.GetDevicePathAndFsType(source)
_, fstype, _, err := utils.GetDevicePathAndFsTypeOptions(source)
if err != nil {
pmemLog.WithError(err).WithField("mount-point", source).Warn("failed to get fstype: using ext4")
fstype = "ext4"
Expand Down
64 changes: 64 additions & 0 deletions virtcontainers/kata_agent.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"syscall"
"time"

units "github.com/docker/go-units"
"github.com/gogo/protobuf/proto"
aTypes "github.com/kata-containers/agent/pkg/types"
kataclient "github.com/kata-containers/agent/protocols/client"
Expand All @@ -33,6 +34,7 @@ import (
"github.com/kata-containers/runtime/virtcontainers/pkg/uuid"
"github.com/kata-containers/runtime/virtcontainers/store"
"github.com/kata-containers/runtime/virtcontainers/types"
"github.com/kata-containers/runtime/virtcontainers/utils"
"github.com/opencontainers/runtime-spec/specs-go"
opentracing "github.com/opentracing/opentracing-go"
"github.com/sirupsen/logrus"
Expand Down Expand Up @@ -1429,6 +1431,10 @@ func (k *kataAgent) createContainer(sandbox *Sandbox, c *Container) (p *Process,
epheStorages := k.handleEphemeralStorage(ociSpec.Mounts)
ctrStorages = append(ctrStorages, epheStorages...)

k.Logger().WithField("ociSpec Hugepage Resources", ociSpec.Linux.Resources.HugepageLimits).Debug("ociSpec HugepageLimit")
hugepages := k.handleHugepages(ociSpec.Mounts, ociSpec.Linux.Resources.HugepageLimits)
ctrStorages = append(ctrStorages, hugepages...)

localStorages := k.handleLocalStorage(ociSpec.Mounts, sandbox.id, c.rootfsSuffix)
ctrStorages = append(ctrStorages, localStorages...)

Expand Down Expand Up @@ -1550,6 +1556,64 @@ func (k *kataAgent) handleEphemeralStorage(mounts []specs.Mount) []*grpc.Storage
return epheStorages
}

// handleHugePages handles hugepages storage by
// creating a Storage from corresponding source of the mount point
func (k *kataAgent) handleHugepages(mounts []specs.Mount, hugepageLimits []specs.LinuxHugepageLimit) []*grpc.Storage {
//Map to hold the total memory of each type of hugepages
optionsMap := make(map[string]string)
var HugePageSizeUnitList = []string{"B", "K", "M", "G", "T", "P"}

for _, hp := range hugepageLimits {
if hp.Limit != 0 {
k.Logger().WithFields(logrus.Fields{
"Pagesize": hp.Pagesize,
"Limit": hp.Limit,
}).Info("hugepage request")
//example Pagesize 2MB, 1GB etc. The Limit are in Bytes
pageSize, _ := units.RAMInBytes(hp.Pagesize)
pageSizeStr := units.CustomSize("%g%s", float64(pageSize), 1024.0, HugePageSizeUnitList)
totalHpSize := fmt.Sprintf("%v", hp.Limit)
optionsMap[pageSizeStr] = totalHpSize
}
}

var hugepages []*grpc.Storage
for idx, mnt := range mounts {
if mnt.Type != KataLocalDevType {
continue
}
//HugePages mount Type is Local
if _, fsType, fsOptions, _ := utils.GetDevicePathAndFsTypeOptions(mnt.Source); fsType == "hugetlbfs" {
k.Logger().WithField("fsOptions", fsOptions).Debug("hugepage mount options")
//Find the pagesize from the mountpoint to use the right set of options
//pagesize is the last element in the mount option. Can it change ?
pagesizeOpt := fsOptions[len(fsOptions)-1]
//Create mount option string
pageSize, _ := units.RAMInBytes(strings.TrimPrefix(pagesizeOpt, "pagesize="))
pageSizeStr := units.CustomSize("%g%s", float64(pageSize), 1024.0, HugePageSizeUnitList)
options := "pagesize=" + pageSizeStr + "," + "size=" + optionsMap[pageSizeStr]
k.Logger().WithField("Hugepage options string", options).Debug("hugepage mount options")
// Set the mount source path to a path that resides inside the VM
mounts[idx].Source = filepath.Join(ephemeralPath(), filepath.Base(mnt.Source))
// Set the mount type to "bind"
mounts[idx].Type = "bind"

// Create a storage struct so that kata agent is able to create
// hugetlbfs backed volume inside the VM
hugepage := &grpc.Storage{
Driver: KataEphemeralDevType,
Source: "nodev",
Fstype: "hugetlbfs",
MountPoint: mounts[idx].Source,
Options: []string{options},
}
hugepages = append(hugepages, hugepage)
}

}
return hugepages
}

// handleLocalStorage handles local storage within the VM
// by creating a directory in the VM from the source of the mount point.
func (k *kataAgent) handleLocalStorage(mounts []specs.Mount, sandboxID string, rootfsSuffix string) []*grpc.Storage {
Expand Down
55 changes: 55 additions & 0 deletions virtcontainers/kata_agent_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -408,6 +408,61 @@ func TestHandleLocalStorage(t *testing.T) {
assert.Equal(t, localMountPoint, expected)
}

func TestHandleHugepages(t *testing.T) {
if os.Getuid() != 0 {
t.Skip("Test disabled as requires root user")
}

dir, err := ioutil.TempDir("", "hugepages-test")
assert.Nil(t, err)
defer os.RemoveAll(dir)

k := kataAgent{}
var mounts []specs.Mount
var hugepageLimits []specs.LinuxHugepageLimit

hugepageDirs := [2]string{"hugepages-1Gi", "hugepages-2Mi"}
options := [2]string{"pagesize=1024M", "pagesize=2M"}

for i := 0; i < 2; i++ {
target := path.Join(dir, hugepageDirs[i])
err := os.MkdirAll(target, 0777)
if err = syscall.Mount("nodev", target, "hugetlbfs", uintptr(0), options[i]); err != nil {
fmt.Printf("Unable to mount %s: %v\n", target, err)
}
assert.Nil(t, err)
defer syscall.Unmount(target, 0)
defer os.RemoveAll(target)
mount := specs.Mount{
Type: KataLocalDevType,
Source: target,
}
mounts = append(mounts, mount)

}
fmt.Printf("mounts: %v\n", mounts)

hugepageLimits = []specs.LinuxHugepageLimit{
{
Pagesize: "1GB",
Limit: 322122547,
},
{
Pagesize: "2MB",
Limit: 134217728,
},
}
fmt.Printf("hugepageLimits: %v\n", hugepageLimits)

hugepages := k.handleHugepages(mounts, hugepageLimits)

fmt.Printf("%v\n", hugepages)

assert.NotNil(t, hugepages)
assert.Equal(t, len(hugepages), 2)

}

func TestHandleDeviceBlockVolume(t *testing.T) {
k := kataAgent{}

Expand Down
4 changes: 2 additions & 2 deletions virtcontainers/mount.go
Original file line number Diff line number Diff line change
Expand Up @@ -400,7 +400,7 @@ func IsEphemeralStorage(path string) bool {
return false
}

if _, fsType, _ := utils.GetDevicePathAndFsType(path); fsType == "tmpfs" {
if _, fsType, _, _ := utils.GetDevicePathAndFsTypeOptions(path); fsType == "tmpfs" {
return true
}

Expand All @@ -415,7 +415,7 @@ func Isk8sHostEmptyDir(path string) bool {
return false
}

if _, fsType, _ := utils.GetDevicePathAndFsType(path); fsType != "tmpfs" {
if _, fsType, _, _ := utils.GetDevicePathAndFsTypeOptions(path); fsType != "tmpfs" {
return true
}
return false
Expand Down
30 changes: 27 additions & 3 deletions virtcontainers/sandbox.go
Original file line number Diff line number Diff line change
Expand Up @@ -1872,7 +1872,10 @@ func (s *Sandbox) updateResources() error {
// Add default vcpus for sandbox
sandboxVCPUs += s.hypervisor.hypervisorConfig().NumVCPUs

sandboxMemoryByte := s.calculateSandboxMemory()
sandboxMemoryByte, err := s.calculateSandboxMemory()
if err != nil {
return err
}
// Add default / rsvd memory for sandbox.
sandboxMemoryByte += int64(s.hypervisor.hypervisorConfig().MemorySize) << utils.MibToBytesShift

Expand Down Expand Up @@ -1912,7 +1915,7 @@ func (s *Sandbox) updateResources() error {
return nil
}

func (s *Sandbox) calculateSandboxMemory() int64 {
func (s *Sandbox) calculateSandboxMemory() (int64, error) {
memorySandbox := int64(0)
for _, c := range s.config.Containers {
// Do not hot add again non-running containers resources
Expand All @@ -1923,9 +1926,30 @@ func (s *Sandbox) calculateSandboxMemory() int64 {

if m := c.Resources.Memory; m != nil && m.Limit != nil {
memorySandbox += *m.Limit
s.Logger().WithField("memory limit", memorySandbox).Info("Memory Sandbox + Memory Limit ")
}

//Add hugepages memory
//HugepageLimit is uint64 - https://github.com/opencontainers/runtime-spec/blob/master/specs-go/config.go#L242
for _, l := range c.Resources.HugepageLimits {
if l.Limit <= math.MaxInt64 {
memorySandbox += int64(l.Limit)
} else {
//Overflow
s.Logger().WithField("hugepages memory limit overflow", l.Limit).Warn("Memory Sandbox + Hugepages Memory Limit ")
return 0, errors.New("Integer overflow")
}

s.Logger().WithField("hugepages memory limit", memorySandbox).Info("Memory Sandbox + Hugepages Memory Limit ")
}
}
return memorySandbox

//Check overflow
if memorySandbox < 0 {
return 0, errors.New("Integer overflow")
} else {
return memorySandbox, nil
}
}

func (s *Sandbox) calculateSandboxCPUs() (uint32, error) {
Expand Down
81 changes: 80 additions & 1 deletion virtcontainers/sandbox_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
"sync"
"syscall"
"testing"
"math"

ktu "github.com/kata-containers/runtime/pkg/katatestutils"
"github.com/kata-containers/runtime/virtcontainers/device/config"
Expand Down Expand Up @@ -167,12 +168,90 @@ func TestCalculateSandboxMem(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
sandbox.config.Containers = tt.containers
got := sandbox.calculateSandboxMemory()
got, err := sandbox.calculateSandboxMemory()
assert.Equal(t, got, tt.want)
assert.NoError(t, err)
})
}
}

func TestSandboxHugepageLimit(t *testing.T) {
contConfig1 := newTestContainerConfigNoop("cont-00001")
contConfig2 := newTestContainerConfigNoop("cont-00002")
limit := int64(4000)
contConfig1.Resources.Memory = &specs.LinuxMemory{Limit: &limit}
contConfig2.Resources.Memory = &specs.LinuxMemory{Limit: &limit}
hConfig := newHypervisorConfig(nil, nil)

defer cleanUp()
// create a sandbox
s, err := testCreateSandbox(t,
testSandboxID,
MockHypervisor,
hConfig,
NoopAgentType,
NetworkConfig{},
[]ContainerConfig{contConfig1, contConfig2},
nil)

assert.NoError(t, err)

hugepageLimits := []specs.LinuxHugepageLimit{
{
Pagesize: "1GB",
Limit: 322122547,
},
{
Pagesize: "2MB",
Limit: 134217728,
},
}

for i := range s.config.Containers {
s.config.Containers[i].Resources.HugepageLimits = hugepageLimits
}
err = s.updateResources()
assert.NoError(t, err)
}


func TestSandboxHugepageLimitOverflow(t *testing.T) {
contConfig1 := newTestContainerConfigNoop("cont-00001")
contConfig2 := newTestContainerConfigNoop("cont-00002")
hConfig := newHypervisorConfig(nil, nil)

defer cleanUp()
// create a sandbox
s, err := testCreateSandbox(t,
testSandboxID,
MockHypervisor,
hConfig,
NoopAgentType,
NetworkConfig{},
[]ContainerConfig{contConfig1, contConfig2},
nil)

assert.NoError(t, err)

hugepageLimits := []specs.LinuxHugepageLimit{
{
Pagesize: "1GB",
Limit: math.MaxUint64,
},
{
Pagesize: "2MB",
Limit: math.MaxUint64,
},
}

for i := range s.config.Containers {
s.config.Containers[i].Resources.HugepageLimits = hugepageLimits
}
err = s.updateResources()
assert.Error(t, err)
}


func TestCreateSandboxEmptyID(t *testing.T) {
hConfig := newHypervisorConfig(nil, nil)
_, err := testCreateSandbox(t, "", MockHypervisor, hConfig, NoopAgentType, NetworkConfig{}, nil, nil)
Expand Down
8 changes: 5 additions & 3 deletions virtcontainers/utils/utils_linux.go
Original file line number Diff line number Diff line change
Expand Up @@ -99,11 +99,12 @@ const (
procDeviceIndex = iota
procPathIndex
procTypeIndex
procOptionIndex
)

// GetDevicePathAndFsType gets the device for the mount point and the file system type
// of the mount.
func GetDevicePathAndFsType(mountPoint string) (devicePath, fsType string, err error) {
// GetDevicePathAndFsTypeOptions gets the device for the mount point, the file system type
// and mount options
func GetDevicePathAndFsTypeOptions(mountPoint string) (devicePath, fsType string, fsOptions []string, err error) {
if mountPoint == "" {
err = fmt.Errorf("Mount point cannot be empty")
return
Expand Down Expand Up @@ -137,6 +138,7 @@ func GetDevicePathAndFsType(mountPoint string) (devicePath, fsType string, err e
if mountPoint == fields[procPathIndex] {
devicePath = fields[procDeviceIndex]
fsType = fields[procTypeIndex]
fsOptions = strings.Split(fields[procOptionIndex], ",")
return
}
}
Expand Down
7 changes: 5 additions & 2 deletions virtcontainers/utils/utils_linux_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,16 +36,19 @@ func TestFindContextID(t *testing.T) {

func TestGetDevicePathAndFsTypeEmptyMount(t *testing.T) {
assert := assert.New(t)
_, _, err := GetDevicePathAndFsType("")
_, _, _, err := GetDevicePathAndFsTypeOptions("")
assert.Error(err)
}

func TestGetDevicePathAndFsTypeSuccessful(t *testing.T) {
assert := assert.New(t)
opts := []string{"rw", "nosuid", "nodev", "noexec", "relatime"}

path, fstype, err := GetDevicePathAndFsType("/proc")
path, fstype, fsOptions, err := GetDevicePathAndFsTypeOptions("/proc")
assert.NoError(err)

assert.Equal(path, "proc")
assert.Equal(fstype, "proc")
assert.True(DeepCompare(fsOptions, opts))
}

0 comments on commit 054ca93

Please sign in to comment.