diff --git a/internal/hcsoci/create.go b/internal/hcsoci/create.go index a71c734bed..331c0dd815 100644 --- a/internal/hcsoci/create.go +++ b/internal/hcsoci/create.go @@ -10,20 +10,24 @@ import ( "os" "path/filepath" "strconv" + "unsafe" "github.com/Microsoft/go-winio/pkg/guid" "github.com/Microsoft/hcsshim/internal/cow" "github.com/Microsoft/hcsshim/internal/guestpath" "github.com/Microsoft/hcsshim/internal/hcs" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + "github.com/Microsoft/hcsshim/internal/jobobject" "github.com/Microsoft/hcsshim/internal/layers" "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/oci" "github.com/Microsoft/hcsshim/internal/resources" "github.com/Microsoft/hcsshim/internal/schemaversion" "github.com/Microsoft/hcsshim/internal/uvm" + "github.com/Microsoft/hcsshim/internal/winapi" specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/sirupsen/logrus" + "golang.org/x/sys/windows" ) var ( @@ -282,9 +286,89 @@ func CreateContainer(ctx context.Context, createOptions *CreateOptions) (_ cow.C if err != nil { return nil, r, err } + + // if container is process isolated, check if affinityCPUs has been set in createOptions. + // If yes, set information on the job object created for this container + if coi.HostingSystem == nil && coi.Spec.Windows != nil { + err = setCPUAffinityOnJobObject(ctx, coi.Spec, system.ID()) + if err != nil { + return nil, r, err + } + } + return system, r, nil } +func setCPUAffinityOnJobObject(ctx context.Context, spec *specs.Spec, computeSystemId string) error { + // + if spec.Windows.Resources == nil || spec.Windows.Resources.CPU == nil || + spec.Windows.Resources.CPU.AffinityCPUs == nil { + return nil + } + + defaultJobObjectName := fmt.Sprintf(`\Container_%s`, computeSystemId) + fmt.Printf("default JO name %v", defaultJobObjectName) + /* + unicodeJobName, err := winapi.NewUnicodeString(defaultJobObjectName) + if err != nil { + return fmt.Errorf("Error getting unicodeJobName %v", err) + } + + jobHandle, err := winapi.OpenJobObject(winapi.JOB_OBJECT_ALL_ACCESS, 0, unicodeJobName.Buffer) + if err != nil { + return fmt.Errorf("Error opening job object %v", err) + } + */ + jobOptions := &jobobject.Options{ + UseNTVariant: true, + Name: defaultJobObjectName, + } + job, err := jobobject.Open(ctx, jobOptions) + if err != nil { + return err + } + defer job.Close() + + // check for numa node number + if spec.Windows.Resources.CPU.AffinityPreferredNumaNodes != nil { + // numberOfPreferredNumaNodes := uint32(len(spec.Windows.Resources.CPU.AffinityPreferredNumaNodes)) + // list of numa nodes might need to be set for this container. Therefore ensure we have enough space + // in attrList for that. + attrList, err := windows.NewProcThreadAttributeList(2) + if err != nil { + return fmt.Errorf("failed to initialize process thread attribute list: %w", err) + } + defer attrList.Delete() + + // Set up the process to only inherit stdio handles and nothing else. + numaNode := (uint16)(spec.Windows.Resources.CPU.AffinityPreferredNumaNodes[0]) + err = attrList.Update( + windows.PROC_THREAD_ATTRIBUTE_PREFERRED_NODE, + unsafe.Pointer(&numaNode), + //uintptr(len(spec.Windows.Resources.CPU.AffinityPreferredNumaNodes))*unsafe.Sizeof(spec.Windows.Resources.CPU.AffinityPreferredNumaNodes[0]), + //uintptr(unsafe.Sizeof(spec.Windows.Resources.CPU.AffinityPreferredNumaNodes[0])), + uintptr(unsafe.Sizeof(numaNode)), + ) + if err != nil { + return fmt.Errorf("Error updating proc thread attribute for numa node, %v", err) + } + + err = job.UpdateProcThreadAttribute(attrList) + if err != nil { + return fmt.Errorf("Error updating proc thread attribute for JO, %v", err) + } + } + + info := make([]winapi.JOBOBJECT_CPU_GROUP_AFFINITY, len(spec.Windows.Resources.CPU.AffinityCPUs)) + for i, cpu := range spec.Windows.Resources.CPU.AffinityCPUs { + info[i].CpuMask = (uintptr)(cpu.CPUMask) + info[i].CpuGroup = (uint16)(cpu.CPUGroup) + info[i].Reserved = [3]uint16{0, 0, 0} + } + + return job.SetInformationJobObject(info) +} + // isV2Xenon returns true if the create options are for a HCS schema V2 xenon container // with a hosting VM func (coi *createOptionsInternal) isV2Xenon() bool { diff --git a/internal/jobobject/jobobject.go b/internal/jobobject/jobobject.go index 10ae4d6700..e0a986c0ff 100644 --- a/internal/jobobject/jobobject.go +++ b/internal/jobobject/jobobject.go @@ -14,6 +14,9 @@ import ( "github.com/Microsoft/hcsshim/internal/queue" "github.com/Microsoft/hcsshim/internal/winapi" + + //"github.com/opencontainers/runtime-spec/specs-go" + "golang.org/x/sys/windows" ) @@ -539,6 +542,43 @@ func isJobSilo(h windows.Handle) bool { return err == nil } +func (job *JobObject) SetInformationJobObject(affinityCPUs []winapi.JOBOBJECT_CPU_GROUP_AFFINITY) error { + len := len(affinityCPUs) + sizeOfGroupAffinity := unsafe.Sizeof(affinityCPUs[0]) + _, err := windows.SetInformationJobObject( + job.handle, + winapi.JobObjectGroupInformationEx, + //uintptr(unsafe.Pointer(affinityCPUs)), + uintptr(unsafe.Pointer(&affinityCPUs[0])), + uint32(uintptr(len)*sizeOfGroupAffinity), + ) + if err != nil { + return fmt.Errorf("failed to set CPU affinities: %w", err) + } + + return nil +} + +func (job *JobObject) GetInformationJobObject() error { + info := make([]winapi.JOBOBJECT_CPU_GROUP_AFFINITY, 10) + var len uint32 + //sizeOfGroupAffinity := unsafe.Sizeof(winapi.JOBOBJECT_CPU_GROUP_AFFINITY{}) + err := windows.QueryInformationJobObject( + job.handle, + (int32)(winapi.JobObjectGroupInformationEx), + //uintptr(unsafe.Pointer(affinityCPUs)), + uintptr(unsafe.Pointer(&info)), + uint32(unsafe.Sizeof(info)*10), + &len, + ) + fmt.Printf("length returned %v", len) + fmt.Printf("data returned %v", info) + if err != nil { + return fmt.Errorf("failed to set CPU affinities: %w", err) + } + return nil +} + // PromoteToSilo promotes a job object to a silo. There must be no running processess // in the job for this to succeed. If the job is already a silo this is a no-op. func (job *JobObject) PromoteToSilo() error { diff --git a/internal/winapi/jobobject.go b/internal/winapi/jobobject.go index b0deb5c72d..319991c97c 100644 --- a/internal/winapi/jobobject.go +++ b/internal/winapi/jobobject.go @@ -55,6 +55,7 @@ const ( JobObjectBasicProcessIdList uint32 = 3 JobObjectBasicAndIoAccountingInformation uint32 = 8 JobObjectLimitViolationInformation uint32 = 13 + JobObjectGroupInformationEx uint32 = 14 JobObjectMemoryUsageInformation uint32 = 28 JobObjectNotificationLimitInformation2 uint32 = 33 JobObjectCreateSilo uint32 = 35 @@ -160,6 +161,13 @@ type JOBOBJECT_ASSOCIATE_COMPLETION_PORT struct { CompletionPort windows.Handle } +// https://learn.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-group_affinity +type JOBOBJECT_CPU_GROUP_AFFINITY struct { + CpuMask uintptr + CpuGroup uint16 + Reserved [3]uint16 +} + // BOOL IsProcessInJob( // HANDLE ProcessHandle, // HANDLE JobHandle,