diff --git a/internal/hcsoci/create.go b/internal/hcsoci/create.go index f87adb1e99..5f16828a65 100644 --- a/internal/hcsoci/create.go +++ b/internal/hcsoci/create.go @@ -220,7 +220,7 @@ func initializeCreateOptions(ctx context.Context, createOptions *CreateOptions) // configureSandboxNetwork creates a new network namespace for the pod (sandbox) // if required and then adds that namespace to the pod. -func configureSandboxNetwork(ctx context.Context, coi *createOptionsInternal, r *resources.Resources) error { +func configureSandboxNetwork(ctx context.Context, coi *createOptionsInternal, r *resources.Resources, ct oci.KubernetesContainerType) error { if coi.NetworkNamespace != "" { r.SetNetNS(coi.NetworkNamespace) } else { @@ -232,15 +232,11 @@ func configureSandboxNetwork(ctx context.Context, coi *createOptionsInternal, r coi.actualNetworkNamespace = r.NetNS() if coi.HostingSystem != nil { - ct, _, err := oci.GetSandboxTypeAndID(coi.Spec.Annotations) - if err != nil { - return err - } // Only add the network namespace to a standalone or sandbox // container but not a workload container in a sandbox that inherits // the namespace. if ct == oci.KubernetesContainerTypeNone || ct == oci.KubernetesContainerTypeSandbox { - if err = SetupNetworkNamespace(ctx, coi.HostingSystem, coi.actualNetworkNamespace); err != nil { + if err := SetupNetworkNamespace(ctx, coi.HostingSystem, coi.actualNetworkNamespace); err != nil { return err } r.SetAddedNetNSToVM(true) @@ -285,13 +281,17 @@ func CreateContainer(ctx context.Context, createOptions *CreateOptions) (_ cow.C } // Create a network namespace if necessary. + ct, _, err := oci.GetSandboxTypeAndID(coi.Spec.Annotations) + if err != nil { + return nil, r, err + } + isSandbox := ct == oci.KubernetesContainerTypeSandbox if coi.Spec.Windows != nil && coi.Spec.Windows.Network != nil && schemaversion.IsV21(coi.actualSchemaVersion) { - err = configureSandboxNetwork(ctx, coi, r) + err = configureSandboxNetwork(ctx, coi, r, ct) if err != nil { return nil, r, fmt.Errorf("failure while creating namespace for container: %s", err) - } } @@ -302,7 +302,7 @@ func CreateContainer(ctx context.Context, createOptions *CreateOptions) (_ cow.C return nil, r, errors.New("LCOW v1 not supported") } log.G(ctx).Debug("hcsshim::CreateContainer allocateLinuxResources") - err = allocateLinuxResources(ctx, coi, r) + err = allocateLinuxResources(ctx, coi, r, isSandbox) if err != nil { log.G(ctx).WithError(err).Debug("failed to allocateLinuxResources") return nil, r, err @@ -313,7 +313,7 @@ func CreateContainer(ctx context.Context, createOptions *CreateOptions) (_ cow.C return nil, r, err } } else { - err = allocateWindowsResources(ctx, coi, r) + err = allocateWindowsResources(ctx, coi, r, isSandbox) if err != nil { log.G(ctx).WithError(err).Debug("failed to allocateWindowsResources") return nil, r, err diff --git a/internal/hcsoci/resources_lcow.go b/internal/hcsoci/resources_lcow.go index 8a1efefcb7..b6b7b40e32 100644 --- a/internal/hcsoci/resources_lcow.go +++ b/internal/hcsoci/resources_lcow.go @@ -35,7 +35,7 @@ func getGPUVHDPath(coi *createOptionsInternal) (string, error) { return gpuVHDPath, nil } -func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r *resources.Resources) error { +func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r *resources.Resources, isSandbox bool) error { if coi.Spec.Root == nil { coi.Spec.Root = &specs.Root{} } @@ -44,10 +44,10 @@ func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r * log.G(ctx).Debug("hcsshim::allocateLinuxResources mounting storage") rootPath, err := layers.MountContainerLayers(ctx, coi.Spec.Windows.LayerFolders, containerRootInUVM, coi.HostingSystem) if err != nil { - return fmt.Errorf("failed to mount container storage: %s", err) + return errors.Wrap(err, "failed to mount container storage") } coi.Spec.Root.Path = rootPath - layers := layers.NewImageLayers(coi.HostingSystem, containerRootInUVM, coi.Spec.Windows.LayerFolders) + layers := layers.NewImageLayers(coi.HostingSystem, containerRootInUVM, coi.Spec.Windows.LayerFolders, isSandbox) r.SetLayers(layers) } else if coi.Spec.Root.Path != "" { // This is the "Plan 9" root filesystem. @@ -56,7 +56,7 @@ func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r * uvmPathForContainersFileSystem := path.Join(r.ContainerRootInUVM(), uvm.RootfsPath) share, err := coi.HostingSystem.AddPlan9(ctx, hostPath, uvmPathForContainersFileSystem, coi.Spec.Root.Readonly, false, nil) if err != nil { - return fmt.Errorf("adding plan9 root: %s", err) + return errors.Wrap(err, "adding plan9 root") } coi.Spec.Root.Path = uvmPathForContainersFileSystem r.Add(share) @@ -95,7 +95,7 @@ func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r * uvmPathForShare = fmt.Sprintf(uvm.LCOWGlobalMountPrefix, coi.HostingSystem.UVMMountCounter()) scsiMount, err := coi.HostingSystem.AddSCSIPhysicalDisk(ctx, hostPath, uvmPathForShare, readOnly) if err != nil { - return fmt.Errorf("adding SCSI physical disk mount %+v: %s", mount, err) + return errors.Wrapf(err, "adding SCSI physical disk mount %+v", mount) } uvmPathForFile = scsiMount.UVMPath @@ -110,7 +110,7 @@ func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r * // that is where it was previously mounted in UVM scsiMount, err := coi.HostingSystem.AddSCSI(ctx, hostPath, uvmPathForShare, readOnly, uvm.VMAccessTypeIndividual) if err != nil { - return fmt.Errorf("adding SCSI virtual disk mount %+v: %s", mount, err) + return errors.Wrapf(err, "adding SCSI virtual disk mount %+v", mount) } uvmPathForFile = scsiMount.UVMPath @@ -124,7 +124,7 @@ func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r * } else { st, err := os.Stat(hostPath) if err != nil { - return fmt.Errorf("could not open bind mount target: %s", err) + return errors.Wrap(err, "could not open bind mount target") } restrictAccess := false var allowedNames []string @@ -140,7 +140,7 @@ func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r * l.Debug("hcsshim::allocateLinuxResources Hot-adding Plan9 for OCI mount") share, err := coi.HostingSystem.AddPlan9(ctx, hostPath, uvmPathForShare, readOnly, restrictAccess, allowedNames) if err != nil { - return fmt.Errorf("adding plan9 mount %+v: %s", mount, err) + return errors.Wrapf(err, "adding plan9 mount %+v", mount) } r.Add(share) } diff --git a/internal/hcsoci/resources_wcow.go b/internal/hcsoci/resources_wcow.go index 0ef3ea5482..83d1a67f24 100644 --- a/internal/hcsoci/resources_wcow.go +++ b/internal/hcsoci/resources_wcow.go @@ -19,11 +19,12 @@ import ( "github.com/Microsoft/hcsshim/internal/uvm" "github.com/Microsoft/hcsshim/internal/wclayer" specs "github.com/opencontainers/runtime-spec/specs-go" + "github.com/pkg/errors" ) -func allocateWindowsResources(ctx context.Context, coi *createOptionsInternal, r *resources.Resources) error { +func allocateWindowsResources(ctx context.Context, coi *createOptionsInternal, r *resources.Resources, isSandbox bool) error { if coi.Spec == nil || coi.Spec.Windows == nil || coi.Spec.Windows.LayerFolders == nil { - return fmt.Errorf("field 'Spec.Windows.Layerfolders' is not populated") + return errors.New("field 'Spec.Windows.Layerfolders' is not populated") } scratchFolder := coi.Spec.Windows.LayerFolders[len(coi.Spec.Windows.LayerFolders)-1] @@ -32,7 +33,7 @@ func allocateWindowsResources(ctx context.Context, coi *createOptionsInternal, r // Create the directory for the RW scratch layer if it doesn't exist if _, err := os.Stat(scratchFolder); os.IsNotExist(err) { if err := os.MkdirAll(scratchFolder, 0777); err != nil { - return fmt.Errorf("failed to auto-create container scratch folder %s: %s", scratchFolder, err) + return errors.Wrapf(err, "failed to auto-create container scratch folder %s", scratchFolder) } } @@ -40,7 +41,7 @@ func allocateWindowsResources(ctx context.Context, coi *createOptionsInternal, r // rather than scratch.vhdx as in the v1 schema, it's hard-coded in HCS. if _, err := os.Stat(filepath.Join(scratchFolder, "sandbox.vhdx")); os.IsNotExist(err) { if err := wclayer.CreateScratchLayer(ctx, scratchFolder, coi.Spec.Windows.LayerFolders[:len(coi.Spec.Windows.LayerFolders)-1]); err != nil { - return fmt.Errorf("failed to CreateSandboxLayer %s", err) + return errors.Wrap(err, "failed to CreateSandboxLayer") } } @@ -53,10 +54,10 @@ func allocateWindowsResources(ctx context.Context, coi *createOptionsInternal, r containerRootInUVM := r.ContainerRootInUVM() containerRootPath, err := layers.MountContainerLayers(ctx, coi.Spec.Windows.LayerFolders, containerRootInUVM, coi.HostingSystem) if err != nil { - return fmt.Errorf("failed to mount container storage: %s", err) + return errors.Wrap(err, "failed to mount container storage") } coi.Spec.Root.Path = containerRootPath - layers := layers.NewImageLayers(coi.HostingSystem, containerRootInUVM, coi.Spec.Windows.LayerFolders) + layers := layers.NewImageLayers(coi.HostingSystem, containerRootInUVM, coi.Spec.Windows.LayerFolders, isSandbox) r.SetLayers(layers) } @@ -136,7 +137,7 @@ func setupMounts(ctx context.Context, coi *createOptionsInternal, r *resources.R l.Debug("hcsshim::allocateWindowsResources Hot-adding SCSI physical disk for OCI mount") scsiMount, err := coi.HostingSystem.AddSCSIPhysicalDisk(ctx, mount.Source, uvmPath, readOnly) if err != nil { - return fmt.Errorf("adding SCSI physical disk mount %+v: %s", mount, err) + return errors.Wrapf(err, "adding SCSI physical disk mount %+v", mount) } coi.Spec.Mounts[i].Type = "" r.Add(scsiMount) @@ -144,7 +145,7 @@ func setupMounts(ctx context.Context, coi *createOptionsInternal, r *resources.R l.Debug("hcsshim::allocateWindowsResources Hot-adding SCSI virtual disk for OCI mount") scsiMount, err := coi.HostingSystem.AddSCSI(ctx, mount.Source, uvmPath, readOnly, uvm.VMAccessTypeIndividual) if err != nil { - return fmt.Errorf("adding SCSI virtual disk mount %+v: %s", mount, err) + return errors.Wrapf(err, "adding SCSI virtual disk mount %+v", mount) } coi.Spec.Mounts[i].Type = "" r.Add(scsiMount) @@ -152,7 +153,7 @@ func setupMounts(ctx context.Context, coi *createOptionsInternal, r *resources.R if uvm.IsPipe(mount.Source) { pipe, err := coi.HostingSystem.AddPipe(ctx, mount.Source) if err != nil { - return fmt.Errorf("failed to add named pipe to UVM: %s", err) + return errors.Wrap(err, "failed to add named pipe to UVM") } r.Add(pipe) } else { @@ -160,7 +161,7 @@ func setupMounts(ctx context.Context, coi *createOptionsInternal, r *resources.R options := coi.HostingSystem.DefaultVSMBOptions(readOnly) share, err := coi.HostingSystem.AddVSMB(ctx, mount.Source, options) if err != nil { - return fmt.Errorf("failed to add VSMB share to utility VM for mount %+v: %s", mount, err) + return errors.Wrapf(err, "failed to add VSMB share to utility VM for mount %+v", mount) } r.Add(share) } @@ -168,5 +169,42 @@ func setupMounts(ctx context.Context, coi *createOptionsInternal, r *resources.R } } + if cs, ok := coi.Spec.Windows.CredentialSpec.(string); ok { + // Only need to create a CCG instance for v2 containers + if schemaversion.IsV21(coi.actualSchemaVersion) { + hypervisorIsolated := coi.HostingSystem != nil + ccgInstance, ccgResource, err := credentials.CreateCredentialGuard(ctx, coi.actualID, cs, hypervisorIsolated) + if err != nil { + return err + } + coi.ccgState = ccgInstance.CredentialGuard + r.Add(ccgResource) + if hypervisorIsolated { + // If hypervisor isolated we need to add an hvsocket service table entry + // By default HVSocket won't allow something inside the VM to connect + // back to a process on the host. We need to update the HVSocket service table + // to allow a connection to CCG.exe on the host, so that GMSA can function. + // We need to hot add this here because at UVM creation time we don't know what containers + // will be launched in the UVM, nonetheless if they will ask for GMSA. This is a workaround + // for the previous design requirement for CCG V2 where the service entry + // must be present in the UVM'S HCS document before being sent over as hot adding + // an HvSocket service was not possible. + hvSockConfig := ccgInstance.HvSocketConfig + if err := coi.HostingSystem.UpdateHvSocketService(ctx, hvSockConfig.ServiceId, hvSockConfig.ServiceConfig); err != nil { + return errors.Wrap(err, "failed to update hvsocket service") + } + } + } + } + + if coi.HostingSystem != nil && coi.hasWindowsAssignedDevices() { + windowsDevices, closers, err := handleAssignedDevicesWindows(ctx, coi.HostingSystem, coi.Spec.Annotations, coi.Spec.Windows.Devices) + if err != nil { + return err + } + r.Add(closers...) + coi.Spec.Windows.Devices = windowsDevices + } + return nil } diff --git a/internal/layers/layers.go b/internal/layers/layers.go index c21234f8c0..5f815b783a 100644 --- a/internal/layers/layers.go +++ b/internal/layers/layers.go @@ -23,18 +23,26 @@ type ImageLayers struct { vm *uvm.UtilityVM containerRootInUVM string layers []string + // In some instances we may want to avoid cleaning up the image layers, such as when tearing + // down a sandbox container since the UVM will be torn down shortly after and the resources + // can be cleaned up on the host. + skipCleanup bool } -func NewImageLayers(vm *uvm.UtilityVM, containerRootInUVM string, layers []string) *ImageLayers { +func NewImageLayers(vm *uvm.UtilityVM, containerRootInUVM string, layers []string, skipCleanup bool) *ImageLayers { return &ImageLayers{ vm: vm, containerRootInUVM: containerRootInUVM, layers: layers, + skipCleanup: skipCleanup, } } // Release unmounts all of the layers located in the layers array. func (layers *ImageLayers) Release(ctx context.Context, all bool) error { + if layers.skipCleanup && layers.vm != nil { + return nil + } op := UnmountOperationSCSI if layers.vm == nil || all { op = UnmountOperationAll @@ -244,9 +252,9 @@ func removeLCOWLayer(ctx context.Context, uvm *uvmpkg.UtilityVM, layerPath strin }).Debug("Removed LCOW layer") return nil } - return fmt.Errorf("failed to remove SCSI layer: %s", err) + return errors.Wrap(err, "failed to remove SCSI layer") } - return fmt.Errorf("failed to remove VPMEM layer: %s", err) + return errors.Wrap(err, "failed to remove VPMEM layer") } // UnmountOperation is used when calling Unmount() to determine what type of unmount is @@ -270,10 +278,10 @@ func UnmountContainerLayers(ctx context.Context, layerFolders []string, containe if uvm == nil { // Must be an argon - folders are mounted on the host if op != UnmountOperationAll { - return fmt.Errorf("only operation supported for host-mounted folders is unmountOperationAll") + return errors.New("only operation supported for host-mounted folders is unmountOperationAll") } if len(layerFolders) < 1 { - return fmt.Errorf("need at least one layer for Unmount") + return errors.New("need at least one layer for Unmount") } path := layerFolders[len(layerFolders)-1] if err := wclayer.UnprepareLayer(ctx, path); err != nil { @@ -286,7 +294,7 @@ func UnmountContainerLayers(ctx context.Context, layerFolders []string, containe // Base+Scratch as a minimum. This is different to v1 which only requires the scratch if len(layerFolders) < 2 { - return fmt.Errorf("at least two layers are required for unmount") + return errors.New("at least two layers are required for unmount") } var retError error @@ -302,7 +310,7 @@ func UnmountContainerLayers(ctx context.Context, layerFolders []string, containe if (op & UnmountOperationSCSI) == UnmountOperationSCSI { hostScratchFile, err := getScratchVHDPath(layerFolders) if err != nil { - return fmt.Errorf("failed to get scratch VHD path in layer folders: %s", err) + return errors.Wrap(err, "failed to get scratch VHD path in layer folders") } if err := uvm.RemoveSCSI(ctx, hostScratchFile); err != nil { log.G(ctx).WithError(err).Warn("failed to remove scratch") @@ -383,7 +391,7 @@ func getScratchVHDPath(layerFolders []string) (string, error) { // Evaluate the symlink here (if there is one). hostPath, err := filepath.EvalSymlinks(hostPath) if err != nil { - return "", fmt.Errorf("failed to eval symlinks: %s", err) + return "", errors.Wrap(err, "failed to eval symlinks") } return hostPath, nil }