Skip to content

Commit

Permalink
SetInformationJO with group_afifnity and numa node
Browse files Browse the repository at this point in the history
Signed-off-by: Kirtana Ashok <kiashok@microsoft.com>
kiashok committed Jun 27, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature.
1 parent 97daa08 commit fba436c
Showing 3 changed files with 132 additions and 0 deletions.
84 changes: 84 additions & 0 deletions internal/hcsoci/create.go
Original file line number Diff line number Diff line change
@@ -10,20 +10,24 @@ import (
"os"
"path/filepath"
"strconv"
"unsafe"

"github.com/Microsoft/go-winio/pkg/guid"
"github.com/Microsoft/hcsshim/internal/cow"
"github.com/Microsoft/hcsshim/internal/guestpath"
"github.com/Microsoft/hcsshim/internal/hcs"
hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2"
"github.com/Microsoft/hcsshim/internal/jobobject"
"github.com/Microsoft/hcsshim/internal/layers"
"github.com/Microsoft/hcsshim/internal/log"
"github.com/Microsoft/hcsshim/internal/oci"
"github.com/Microsoft/hcsshim/internal/resources"
"github.com/Microsoft/hcsshim/internal/schemaversion"
"github.com/Microsoft/hcsshim/internal/uvm"
"github.com/Microsoft/hcsshim/internal/winapi"
specs "github.com/opencontainers/runtime-spec/specs-go"
"github.com/sirupsen/logrus"
"golang.org/x/sys/windows"
)

var (
@@ -282,9 +286,89 @@ func CreateContainer(ctx context.Context, createOptions *CreateOptions) (_ cow.C
if err != nil {
return nil, r, err
}

// if container is process isolated, check if affinityCPUs has been set in createOptions.
// If yes, set information on the job object created for this container
if coi.HostingSystem == nil && coi.Spec.Windows != nil {
err = setCPUAffinityOnJobObject(ctx, coi.Spec, system.ID())
if err != nil {
return nil, r, err
}
}

return system, r, nil
}

func setCPUAffinityOnJobObject(ctx context.Context, spec *specs.Spec, computeSystemId string) error {
//
if spec.Windows.Resources == nil || spec.Windows.Resources.CPU == nil ||
spec.Windows.Resources.CPU.AffinityCPUs == nil {
return nil
}

defaultJobObjectName := fmt.Sprintf(`\Container_%s`, computeSystemId)
fmt.Printf("default JO name %v", defaultJobObjectName)
/*
unicodeJobName, err := winapi.NewUnicodeString(defaultJobObjectName)
if err != nil {
return fmt.Errorf("Error getting unicodeJobName %v", err)
}
jobHandle, err := winapi.OpenJobObject(winapi.JOB_OBJECT_ALL_ACCESS, 0, unicodeJobName.Buffer)
if err != nil {
return fmt.Errorf("Error opening job object %v", err)
}
*/
jobOptions := &jobobject.Options{
UseNTVariant: true,
Name: defaultJobObjectName,
}
job, err := jobobject.Open(ctx, jobOptions)
if err != nil {
return err
}
defer job.Close()

// check for numa node number
if spec.Windows.Resources.CPU.AffinityPreferredNumaNodes != nil {
// numberOfPreferredNumaNodes := uint32(len(spec.Windows.Resources.CPU.AffinityPreferredNumaNodes))
// list of numa nodes might need to be set for this container. Therefore ensure we have enough space
// in attrList for that.
attrList, err := windows.NewProcThreadAttributeList(2)
if err != nil {
return fmt.Errorf("failed to initialize process thread attribute list: %w", err)
}
defer attrList.Delete()

// Set up the process to only inherit stdio handles and nothing else.
numaNode := (uint16)(spec.Windows.Resources.CPU.AffinityPreferredNumaNodes[0])
err = attrList.Update(
windows.PROC_THREAD_ATTRIBUTE_PREFERRED_NODE,
unsafe.Pointer(&numaNode),
//uintptr(len(spec.Windows.Resources.CPU.AffinityPreferredNumaNodes))*unsafe.Sizeof(spec.Windows.Resources.CPU.AffinityPreferredNumaNodes[0]),
//uintptr(unsafe.Sizeof(spec.Windows.Resources.CPU.AffinityPreferredNumaNodes[0])),
uintptr(unsafe.Sizeof(numaNode)),
)
if err != nil {
return fmt.Errorf("Error updating proc thread attribute for numa node, %v", err)
}

err = job.UpdateProcThreadAttribute(attrList)
if err != nil {
return fmt.Errorf("Error updating proc thread attribute for JO, %v", err)
}
}

info := make([]winapi.JOBOBJECT_CPU_GROUP_AFFINITY, len(spec.Windows.Resources.CPU.AffinityCPUs))
for i, cpu := range spec.Windows.Resources.CPU.AffinityCPUs {
info[i].CpuMask = (uintptr)(cpu.CPUMask)
info[i].CpuGroup = (uint16)(cpu.CPUGroup)
info[i].Reserved = [3]uint16{0, 0, 0}
}

return job.SetInformationJobObject(info)
}

// isV2Xenon returns true if the create options are for a HCS schema V2 xenon container
// with a hosting VM
func (coi *createOptionsInternal) isV2Xenon() bool {
40 changes: 40 additions & 0 deletions internal/jobobject/jobobject.go
Original file line number Diff line number Diff line change
@@ -14,6 +14,9 @@ import (

"github.com/Microsoft/hcsshim/internal/queue"
"github.com/Microsoft/hcsshim/internal/winapi"

//"github.com/opencontainers/runtime-spec/specs-go"

"golang.org/x/sys/windows"
)

@@ -539,6 +542,43 @@ func isJobSilo(h windows.Handle) bool {
return err == nil
}

func (job *JobObject) SetInformationJobObject(affinityCPUs []winapi.JOBOBJECT_CPU_GROUP_AFFINITY) error {
len := len(affinityCPUs)
sizeOfGroupAffinity := unsafe.Sizeof(affinityCPUs[0])
_, err := windows.SetInformationJobObject(
job.handle,
winapi.JobObjectGroupInformationEx,
//uintptr(unsafe.Pointer(affinityCPUs)),
uintptr(unsafe.Pointer(&affinityCPUs[0])),
uint32(uintptr(len)*sizeOfGroupAffinity),
)
if err != nil {
return fmt.Errorf("failed to set CPU affinities: %w", err)
}

return nil
}

func (job *JobObject) GetInformationJobObject() error {
info := make([]winapi.JOBOBJECT_CPU_GROUP_AFFINITY, 10)
var len uint32
//sizeOfGroupAffinity := unsafe.Sizeof(winapi.JOBOBJECT_CPU_GROUP_AFFINITY{})
err := windows.QueryInformationJobObject(
job.handle,
(int32)(winapi.JobObjectGroupInformationEx),
//uintptr(unsafe.Pointer(affinityCPUs)),
uintptr(unsafe.Pointer(&info)),
uint32(unsafe.Sizeof(info)*10),
&len,
)
fmt.Printf("length returned %v", len)
fmt.Printf("data returned %v", info)
if err != nil {
return fmt.Errorf("failed to set CPU affinities: %w", err)
}
return nil
}

// PromoteToSilo promotes a job object to a silo. There must be no running processess
// in the job for this to succeed. If the job is already a silo this is a no-op.
func (job *JobObject) PromoteToSilo() error {
8 changes: 8 additions & 0 deletions internal/winapi/jobobject.go
Original file line number Diff line number Diff line change
@@ -55,6 +55,7 @@ const (
JobObjectBasicProcessIdList uint32 = 3
JobObjectBasicAndIoAccountingInformation uint32 = 8
JobObjectLimitViolationInformation uint32 = 13
JobObjectGroupInformationEx uint32 = 14
JobObjectMemoryUsageInformation uint32 = 28
JobObjectNotificationLimitInformation2 uint32 = 33
JobObjectCreateSilo uint32 = 35
@@ -160,6 +161,13 @@ type JOBOBJECT_ASSOCIATE_COMPLETION_PORT struct {
CompletionPort windows.Handle
}

// https://learn.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-group_affinity
type JOBOBJECT_CPU_GROUP_AFFINITY struct {
CpuMask uintptr
CpuGroup uint16
Reserved [3]uint16
}

// BOOL IsProcessInJob(
// HANDLE ProcessHandle,
// HANDLE JobHandle,

0 comments on commit fba436c

Please sign in to comment.