Merge pull request #820 from microsoft/user/ambarve/external_gcs_wcow

Enable external GCS connection for WCOW.

internal/gcs/protocol.go

@@ -22,6 +22,14 @@ var WindowsGcsHvsockServiceID = guid.GUID{
 	Data4: [8]uint8{0x85, 0x6b, 0x62, 0x45, 0xe6, 0x9f, 0x46, 0x20},
 }
 
+// WindowsGcsHvHostID is the hvsock address for the parent of the VM running the GCS
+var WindowsGcsHvHostID = guid.GUID{
+	Data1: 0x894cc2d6,
+	Data2: 0x9d79,
+	Data3: 0x424f,
+	Data4: [8]uint8{0x93, 0xfe, 0x42, 0x96, 0x9a, 0xe6, 0xd8, 0xd1},
+}
+
 type anyInString struct {
 	Value interface{}
 }

internal/guestrequest/types.go

@@ -77,6 +77,7 @@ const (
 	ResourceTypeCombinedLayers    ResourceType = "CombinedLayers"
 	ResourceTypeVPMemDevice       ResourceType = "VPMemDevice"
 	ResourceTypeVPCIDevice        ResourceType = "VPCIDevice"
+	ResourceTypeHvSocket          ResourceType = "HvSocket"
 )
 
 // GuestRequest is for modify commands passed to the guest.

internal/oci/uvm.go

@@ -124,6 +124,10 @@ const (
 	annotationStorageQoSBandwidthMaximum = "io.microsoft.virtualmachine.storageqos.bandwidthmaximum"
 	annotationStorageQoSIopsMaximum      = "io.microsoft.virtualmachine.storageqos.iopsmaximum"
 	annotationFullyPhysicallyBacked      = "io.microsoft.virtualmachine.fullyphysicallybacked"
+	// A boolean annotation to control whether to use an external bridge or the
+	// HCS-GCS bridge. Default value is true which means external bridge will be used
+	// by default.
+	annotationUseExternalGCSBridge = "io.microsoft.virtualmachine.useexternalgcsbridge"
 )
 
 // parseAnnotationsBool searches `a` for `key` and if found verifies that the
@@ -379,6 +383,7 @@ func SpecToUVMCreateOpts(ctx context.Context, s *specs.Spec, id, owner string) (
 		lopts.StorageQoSIopsMaximum = ParseAnnotationsStorageIops(ctx, s, annotationStorageQoSIopsMaximum, lopts.StorageQoSIopsMaximum)
 		lopts.VPCIEnabled = parseAnnotationsBool(ctx, s.Annotations, annotationVPCIEnabled, lopts.VPCIEnabled)
 		lopts.BootFilesPath = parseAnnotationsString(s.Annotations, annotationBootFilesRootPath, lopts.BootFilesPath)
+		lopts.ExternalGuestConnection = parseAnnotationsBool(ctx, s.Annotations, annotationUseExternalGCSBridge, lopts.ExternalGuestConnection)
 		handleAnnotationPreferredRootFSType(ctx, s.Annotations, lopts)
 		handleAnnotationKernelDirectBoot(ctx, s.Annotations, lopts)
 
@@ -399,6 +404,7 @@ func SpecToUVMCreateOpts(ctx context.Context, s *specs.Spec, id, owner string) (
 		wopts.ProcessorWeight = ParseAnnotationsCPUWeight(ctx, s, annotationProcessorWeight, wopts.ProcessorWeight)
 		wopts.StorageQoSBandwidthMaximum = ParseAnnotationsStorageBps(ctx, s, annotationStorageQoSBandwidthMaximum, wopts.StorageQoSBandwidthMaximum)
 		wopts.StorageQoSIopsMaximum = ParseAnnotationsStorageIops(ctx, s, annotationStorageQoSIopsMaximum, wopts.StorageQoSIopsMaximum)
+		wopts.ExternalGuestConnection = parseAnnotationsBool(ctx, s.Annotations, annotationUseExternalGCSBridge, wopts.ExternalGuestConnection)
 		handleAnnotationFullyPhysicallyBacked(ctx, s.Annotations, wopts)
 		return wopts, nil
 	}

internal/schema2/hv_socket_address.go

@@ -0,0 +1,17 @@
+/*
+ * HCS API
+ *
+ * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)
+ *
+ * API version: 2.4
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+
+package hcsschema
+
+//  This class defines address settings applied to a VM
+//  by the GCS every time a VM starts or restores.
+type HvSocketAddress struct {
+	LocalAddress  string `json:"LocalAddress,omitempty"`
+	ParentAddress string `json:"ParentAddress,omitempty"`
+}

internal/uvm/create.go

@@ -119,13 +119,14 @@ func verifyOptions(ctx context.Context, options interface{}) error {
 // If `owner` is empty it will be set to the calling executables name.
 func newDefaultOptions(id, owner string) *Options {
 	opts := &Options{
-		ID:                    id,
-		Owner:                 owner,
-		MemorySizeInMB:        1024,
-		AllowOvercommit:       true,
-		EnableDeferredCommit:  false,
-		ProcessorCount:        defaultProcessorCount(),
-		FullyPhysicallyBacked: false,
+		ID:                      id,
+		Owner:                   owner,
+		MemorySizeInMB:          1024,
+		AllowOvercommit:         true,
+		EnableDeferredCommit:    false,
+		ProcessorCount:          defaultProcessorCount(),
+		ExternalGuestConnection: true,
+		FullyPhysicallyBacked:   false,
 	}
 
 	if opts.Owner == "" {

internal/uvm/create_lcow.go

@@ -116,9 +116,6 @@ func NewDefaultOptionsLCOW(id, owner string) *OptionsLCOW {
 		VPCIEnabled:           false,
 	}
 
-	// LCOW has more reliable behavior with the external bridge.
-	opts.Options.ExternalGuestConnection = true
-
 	if _, err := os.Stat(filepath.Join(opts.BootFilesPath, VhdFile)); err == nil {
 		// We have a rootfs.vhd in the boot files path. Use it over an initrd.img
 		opts.RootFSFile = VhdFile
@@ -381,7 +378,7 @@ func CreateLCOW(ctx context.Context, opts *OptionsLCOW) (_ *UtilityVM, err error
 
 	err = uvm.create(ctx, fullDoc)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("error while creating the compute system: %s", err)
 	}
 
 	// Cerate a socket to inject entropy during boot.
@@ -402,6 +399,7 @@ func CreateLCOW(ctx context.Context, opts *OptionsLCOW) (_ *UtilityVM, err error
 	}
 
 	if opts.UseGuestConnection && opts.ExternalGuestConnection {
+		log.G(ctx).WithField("vmID", uvm.runtimeID).Debug("Using external GCS bridge")
 		l, err := uvm.listenVsock(gcs.LinuxGcsVsockPort)
 		if err != nil {
 			return nil, err

internal/uvm/create_wcow.go

@@ -221,10 +221,11 @@ func CreateWCOW(ctx context.Context, opts *OptionsWCOW) (_ *UtilityVM, err error
 
 	err = uvm.create(ctx, fullDoc)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("error while creating the compute system: %s", err)
 	}
 
 	if opts.ExternalGuestConnection {
+		log.G(ctx).WithField("vmID", uvm.runtimeID).Debug("Using external GCS bridge")
 		l, err := winio.ListenHvsock(&winio.HvsockAddr{
 			VMID:      uvm.runtimeID,
 			ServiceID: gcs.WindowsGcsHvsockServiceID,

internal/uvm/start.go

@@ -14,9 +14,12 @@ import (
 	"time"
 
 	"github.com/Microsoft/hcsshim/internal/gcs"
+	"github.com/Microsoft/hcsshim/internal/guestrequest"
 	"github.com/Microsoft/hcsshim/internal/log"
 	"github.com/Microsoft/hcsshim/internal/logfields"
+	"github.com/Microsoft/hcsshim/internal/requesttype"
 	"github.com/Microsoft/hcsshim/internal/schema1"
+	hcsschema "github.com/Microsoft/hcsshim/internal/schema2"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/sync/errgroup"
 )
@@ -115,6 +118,37 @@ func parseLogrus(vmid string) func(r io.Reader) {
 	}
 }
 
+// When using an external GCS connection it is necessary to send a ModifySettings request
+// for HvSockt so that the GCS can setup some registry keys that are required for running
+// containers inside the UVM. In non external GCS connection scenarios this is done by the
+// HCS immediately after the GCS connection is done. Since, we are using the external GCS
+// connection we should do that setup here after we connect with the GCS.
+// This only applies for WCOW
+func (uvm *UtilityVM) configureHvSocketForGCS(ctx context.Context) (err error) {
+	if uvm.OS() != "windows" {
+		return nil
+	}
+
+	hvsocketAddress := &hcsschema.HvSocketAddress{
+		LocalAddress:  uvm.runtimeID.String(),
+		ParentAddress: gcs.WindowsGcsHvHostID.String(),
+	}
+
+	conSetupReq := &hcsschema.ModifySettingRequest{
+		GuestRequest: guestrequest.GuestRequest{
+			RequestType:  requesttype.Update,
+			ResourceType: guestrequest.ResourceTypeHvSocket,
+			Settings:     hvsocketAddress,
+		},
+	}
+
+	if err = uvm.modify(ctx, conSetupReq); err != nil {
+		return fmt.Errorf("failed to configure HVSOCK for external GCS: %s", err)
+	}
+
+	return nil
+}
+
 // Start synchronously starts the utility VM.
 func (uvm *UtilityVM) Start(ctx context.Context) (err error) {
 	ctx, cancel := context.WithTimeout(ctx, 2*time.Minute)
@@ -205,6 +239,11 @@ func (uvm *UtilityVM) Start(ctx context.Context) (err error) {
 		}
 		uvm.guestCaps = *uvm.gc.Capabilities()
 		uvm.protocol = uvm.gc.Protocol()
+
+		// initial setup required for external GCS connection
+		if err = uvm.configureHvSocketForGCS(ctx); err != nil {
+			return fmt.Errorf("failed to do initial GCS setup: %s", err)
+		}
 	} else {
 		// Cache the guest connection properties.
 		properties, err := uvm.hcsSystem.Properties(ctx, schema1.PropertyTypeGuestConnection)

test/cri-containerd/runpodsandbox_test.go

@@ -67,6 +67,27 @@ func Test_RunPodSandbox_WCOW_Hypervisor(t *testing.T) {
 	runPodSandboxTest(t, request)
 }
 
+func Test_RunPodSandbox_WCOW_Hypervisor_WithoutExternalBridge(t *testing.T) {
+	requireFeatures(t, featureWCOWHypervisor)
+
+	pullRequiredImages(t, []string{imageWindowsNanoserver})
+
+	request := &runtime.RunPodSandboxRequest{
+		Config: &runtime.PodSandboxConfig{
+			Metadata: &runtime.PodSandboxMetadata{
+				Name:      t.Name(),
+				Uid:       "0",
+				Namespace: testNamespace,
+			},
+			Annotations: map[string]string{
+				"io.microsoft.virtualmachine.useexternalgcsbridge": "false",
+			},
+		},
+		RuntimeHandler: wcowHypervisorRuntimeHandler,
+	}
+	runPodSandboxTest(t, request)
+}
+
 func Test_RunPodSandbox_LCOW(t *testing.T) {
 	requireFeatures(t, featureLCOW)