Add support for assinging VMs to job objects.

This commit adds a couple of annotations to pass named job
objects (memory and CPU), which can be used to query corresponding
job object handles and assigning VMs to them.

Signed-off-by: Maksim An <maksiman@microsoft.com>

container.go

@@ -78,7 +78,7 @@ func CreateContainer(id string, c *ContainerConfig) (Container, error) {
  return nil, fmt.Errorf("failed to merge additional JSON '%s': %w", createContainerAdditionalJSON, err)
  }
 
- system, err := hcs.CreateComputeSystem(context.Background(), id, fullConfig)
+ system, err := hcs.CreateComputeSystem(context.Background(), id, fullConfig, nil)
  if err != nil {
  return nil, err
  }

internal/annotations/annotations.go

@@ -21,4 +21,10 @@ const (
 
  // ExtraVSockPorts adds additional ports to the list of ports that the UVM is allowed to use.
  ExtraVSockPorts = "io.microsoft.virtualmachine.lcow.extra-vsock-ports"
+
+ // HRMCPUJobName specifies the job name of the HRM CPU job object that the underlying HCS system should be bound to.
+ HRMCPUJobName = "io.microsoft.virtualmachine.hrm.cpu-job-name"
+
+ // HRMMemoryJobName specifies the job name of the HRM memory job object that the underlying HCS system should be bound to.
+ HRMMemoryJobName = "io.microsoft.virtualmachine.hrm.memory-job-name"
 )

internal/computecore/computecore.go

@@ -33,14 +33,14 @@ type (
 
 func HcsCreateComputeSystem(ctx context.Context, id string, configuration string, operation HCSOperation, securityDescriptor *uint32) (computeSystem HCSSystem, result string, hr error) {
  ctx, span := oc.StartSpan(ctx, "computecore::HcsCreateComputeSystem")
- defer span.End()
  defer func() {
  if result != "" {
  span.AddAttributes(trace.StringAttribute("resultDocument", result))
  }
  if !errors.Is(hr, windows.ERROR_VMCOMPUTE_OPERATION_PENDING) {
  oc.SetSpanStatus(span, hr)
  }
+ span.End()
  }()
  span.AddAttributes(
  trace.StringAttribute("id", id),

internal/hcs/system.go

@@ -7,6 +7,7 @@ import (
  "encoding/json"
  "errors"
  "fmt"
+ "github.com/Microsoft/hcsshim/internal/computecore"
  "strings"
  "sync"
  "syscall"
@@ -28,6 +29,7 @@ import (
 type System struct {
  handleLock sync.RWMutex
  handle vmcompute.HcsSystem
+ handleV2 computecore.HCSSystem
  id string
  callbackNumber uintptr
 
@@ -39,6 +41,11 @@ type System struct {
  startTime time.Time
 }
 
+type ResourcePoolOptions struct {
+ MemoryPoolJobName string
+ CPUPoolJobName string
+}
+
 var _ cow.Container = &System{}
 var _ cow.ProcessHost = &System{}
 
@@ -55,7 +62,7 @@ func siloNameFmt(containerID string) string {
 }
 
 // CreateComputeSystem creates a new compute system with the given configuration but does not start it.
-func CreateComputeSystem(ctx context.Context, id string, hcsDocumentInterface interface{}) (_ *System, err error) {
+func CreateComputeSystem(ctx context.Context, id string, hcsDocumentInterface interface{}, rpOptions *ResourcePoolOptions) (_ *System, err error) {
  operation := "hcs::CreateComputeSystem"
 
  // hcsCreateComputeSystemContext is an async operation. Start the outer span
@@ -75,12 +82,42 @@ func CreateComputeSystem(ctx context.Context, id string, hcsDocumentInterface in
  hcsDocument := string(hcsDocumentB)
 
  var (
- identity syscall.Handle
- resultJSON string
- createError error
+ //identity syscall.Handle
+ hcsOp computecore.HCSOperation
+ resultJSON string
+ createError error
+ operationOpts computecore.CreateOperationOptions
  )
- computeSystem.handle, resultJSON, createError = vmcompute.HcsCreateComputeSystem(ctx, id, hcsDocument, identity)
+ if rpOptions != nil {
+ // If resource pool configuration isn't empty, we should be using HCS v2
+ // APIs for creating the compute system.
+ if rpOptions.MemoryPoolJobName != "" {
+ operationOpts.JobResources = append(operationOpts.JobResources, computecore.HCSJobResource{
+ Name: rpOptions.MemoryPoolJobName,
+ Uri: computecore.HCSMemoryJobUri,
+ })
+ }
+ if rpOptions.CPUPoolJobName != "" {
+ operationOpts.JobResources = append(operationOpts.JobResources, computecore.HCSJobResource{
+ Name: rpOptions.CPUPoolJobName,
+ Uri: computecore.HCSCpuJobUri,
+ })
+ }
+ }
+ hcsOp, err = computecore.HcsCreateOperation(ctx, 0, 0, &operationOpts)
+ if err != nil {
+ return nil, err
+ }
+ defer func() {
+ if iErr := hcsOp.Close(); iErr != nil {
+ log.G(ctx).WithError(iErr).Error("failed to close HCS operation")
+ }
+ }()
+
+ //computeSystem.handle, resultJSON, createError = vmcompute.HcsCreateComputeSystem(ctx, id, hcsDocument, identity)
+ computeSystem.handleV2, resultJSON, createError = computecore.HcsCreateComputeSystem(ctx, id, hcsDocument, hcsOp, nil)
  if createError == nil || IsPending(createError) {
+ computeSystem.handle = vmcompute.HcsSystem(computeSystem.handleV2)
  defer func() {
  if err != nil {
  computeSystem.Close()
@@ -122,6 +159,8 @@ func OpenComputeSystem(ctx context.Context, id string) (*System, error) {
  return nil, makeSystemError(computeSystem, operation, err, events)
  }
  computeSystem.handle = handle
+ // FIXME: is this needed?
+ computeSystem.handleV2 = computecore.HCSSystem(handle)
  defer func() {
  if err != nil {
  computeSystem.Close()
@@ -359,7 +398,7 @@ func (computeSystem *System) Properties(ctx context.Context, types ...schema1.Pr
  return nil, makeSystemError(computeSystem, operation, err, nil)
  }
 
- propertiesJSON, resultJSON, err := vmcompute.HcsGetComputeSystemProperties(ctx, computeSystem.handle, string(queryBytes))
+ propertiesJSON, resultJSON, err := vmcompute.HcsGetComputeSystemProperties(ctx, vmcompute.HcsSystem(computeSystem.handle), string(queryBytes))
  events := processHcsResult(ctx, resultJSON)
  if err != nil {
  return nil, makeSystemError(computeSystem, operation, err, events)

internal/hcsoci/create.go

@@ -278,7 +278,7 @@ func CreateContainer(ctx context.Context, createOptions *CreateOptions) (_ cow.C
  return c, r, nil
  }
 
- system, err := hcs.CreateComputeSystem(ctx, coi.actualID, hcsDocument)
+ system, err := hcs.CreateComputeSystem(ctx, coi.actualID, hcsDocument, nil)
  if err != nil {
  return nil, r, err
  }

internal/uvm/create.go

@@ -180,7 +180,7 @@ func (uvm *UtilityVM) OS() string {
 
 func (uvm *UtilityVM) create(ctx context.Context, doc interface{}) error {
  uvm.exitCh = make(chan struct{})
- system, err := hcs.CreateComputeSystem(ctx, uvm.id, doc)
+ system, err := hcs.CreateComputeSystem(ctx, uvm.id, doc, nil)
  if err != nil {
  return err
  }
@@ -285,7 +285,7 @@ func (uvm *UtilityVM) CreateContainer(ctx context.Context, id string, settings i
  ShouldTerminateOnLastHandleClosed: true,
  HostedSystem: settings,
  }
- c, err := hcs.CreateComputeSystem(ctx, id, &doc)
+ c, err := hcs.CreateComputeSystem(ctx, id, &doc, nil)
  if err != nil {
  return nil, err
  }