Skip to content

Commit

Permalink
Skip unmount layers for sandbox container.
Browse files Browse the repository at this point in the history 
* Skip unmounting the layers for the sandbox container as we know the UVM
gets torn down shortly afterwards.

Signed-off-by: Daniel Canter <dcanter@microsoft.com>
  • Loading branch information
@dcantah
dcantah committed Jan 27, 2021
1 parent e7d50a7 commit d221e81
Show file tree
Hide file tree
Showing 4 changed files with 82 additions and 36 deletions.
20 changes: 10 additions & 10 deletions internal/hcsoci/create.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -220,7 +220,7 @@ func initializeCreateOptions(ctx context.Context, createOptions *CreateOptions)

// configureSandboxNetwork creates a new network namespace for the pod (sandbox)
// if required and then adds that namespace to the pod.
func configureSandboxNetwork(ctx context.Context, coi *createOptionsInternal, r *resources.Resources) error {
func configureSandboxNetwork(ctx context.Context, coi *createOptionsInternal, r *resources.Resources, ct oci.KubernetesContainerType) error {
if coi.NetworkNamespace != "" {
r.SetNetNS(coi.NetworkNamespace)
} else {
Expand All @@ -232,15 +232,11 @@ func configureSandboxNetwork(ctx context.Context, coi *createOptionsInternal, r
coi.actualNetworkNamespace = r.NetNS()

if coi.HostingSystem != nil {
ct, _, err := oci.GetSandboxTypeAndID(coi.Spec.Annotations)
if err != nil {
return err
}
// Only add the network namespace to a standalone or sandbox
// container but not a workload container in a sandbox that inherits
// the namespace.
if ct == oci.KubernetesContainerTypeNone || ct == oci.KubernetesContainerTypeSandbox {
if err = SetupNetworkNamespace(ctx, coi.HostingSystem, coi.actualNetworkNamespace); err != nil {
if err := SetupNetworkNamespace(ctx, coi.HostingSystem, coi.actualNetworkNamespace); err != nil {
return err
}
r.SetAddedNetNSToVM(true)
Expand Down Expand Up @@ -285,13 +281,17 @@ func CreateContainer(ctx context.Context, createOptions *CreateOptions) (_ cow.C
}

// Create a network namespace if necessary.
ct, _, err := oci.GetSandboxTypeAndID(coi.Spec.Annotations)
if err != nil {
return nil, r, err
}
isSandbox := ct == oci.KubernetesContainerTypeSandbox
if coi.Spec.Windows != nil &&
coi.Spec.Windows.Network != nil &&
schemaversion.IsV21(coi.actualSchemaVersion) {
err = configureSandboxNetwork(ctx, coi, r)
err = configureSandboxNetwork(ctx, coi, r, ct)
if err != nil {
return nil, r, fmt.Errorf("failure while creating namespace for container: %s", err)

}
}

Expand All @@ -302,7 +302,7 @@ func CreateContainer(ctx context.Context, createOptions *CreateOptions) (_ cow.C
return nil, r, errors.New("LCOW v1 not supported")
}
log.G(ctx).Debug("hcsshim::CreateContainer allocateLinuxResources")
err = allocateLinuxResources(ctx, coi, r)
err = allocateLinuxResources(ctx, coi, r, isSandbox)
if err != nil {
log.G(ctx).WithError(err).Debug("failed to allocateLinuxResources")
return nil, r, err
Expand All @@ -313,7 +313,7 @@ func CreateContainer(ctx context.Context, createOptions *CreateOptions) (_ cow.C
return nil, r, err
}
} else {
err = allocateWindowsResources(ctx, coi, r)
err = allocateWindowsResources(ctx, coi, r, isSandbox)
if err != nil {
log.G(ctx).WithError(err).Debug("failed to allocateWindowsResources")
return nil, r, err
Expand Down
16 changes: 8 additions & 8 deletions internal/hcsoci/resources_lcow.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ func getGPUVHDPath(coi *createOptionsInternal) (string, error) {
return gpuVHDPath, nil
}

func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r *resources.Resources) error {
func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r *resources.Resources, isSandbox bool) error {
if coi.Spec.Root == nil {
coi.Spec.Root = &specs.Root{}
}
Expand All @@ -44,10 +44,10 @@ func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r *
log.G(ctx).Debug("hcsshim::allocateLinuxResources mounting storage")
rootPath, err := layers.MountContainerLayers(ctx, coi.Spec.Windows.LayerFolders, containerRootInUVM, coi.HostingSystem)
if err != nil {
return fmt.Errorf("failed to mount container storage: %s", err)
return errors.Wrap(err, "failed to mount container storage")
}
coi.Spec.Root.Path = rootPath
layers := layers.NewImageLayers(coi.HostingSystem, containerRootInUVM, coi.Spec.Windows.LayerFolders)
layers := layers.NewImageLayers(coi.HostingSystem, containerRootInUVM, coi.Spec.Windows.LayerFolders, isSandbox)
r.SetLayers(layers)
} else if coi.Spec.Root.Path != "" {
// This is the "Plan 9" root filesystem.
Expand All @@ -56,7 +56,7 @@ func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r *
uvmPathForContainersFileSystem := path.Join(r.ContainerRootInUVM(), uvm.RootfsPath)
share, err := coi.HostingSystem.AddPlan9(ctx, hostPath, uvmPathForContainersFileSystem, coi.Spec.Root.Readonly, false, nil)
if err != nil {
return fmt.Errorf("adding plan9 root: %s", err)
return errors.Wrap(err, "adding plan9 root")
}
coi.Spec.Root.Path = uvmPathForContainersFileSystem
r.Add(share)
Expand Down Expand Up @@ -95,7 +95,7 @@ func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r *
uvmPathForShare = fmt.Sprintf(uvm.LCOWGlobalMountPrefix, coi.HostingSystem.UVMMountCounter())
scsiMount, err := coi.HostingSystem.AddSCSIPhysicalDisk(ctx, hostPath, uvmPathForShare, readOnly)
if err != nil {
return fmt.Errorf("adding SCSI physical disk mount %+v: %s", mount, err)
return errors.Wrapf(err, "adding SCSI physical disk mount %+v", mount)
}

uvmPathForFile = scsiMount.UVMPath
Expand All @@ -110,7 +110,7 @@ func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r *
// that is where it was previously mounted in UVM
scsiMount, err := coi.HostingSystem.AddSCSI(ctx, hostPath, uvmPathForShare, readOnly, uvm.VMAccessTypeIndividual)
if err != nil {
return fmt.Errorf("adding SCSI virtual disk mount %+v: %s", mount, err)
return errors.Wrapf(err, "adding SCSI virtual disk mount %+v", mount)
}

uvmPathForFile = scsiMount.UVMPath
Expand All @@ -124,7 +124,7 @@ func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r *
} else {
st, err := os.Stat(hostPath)
if err != nil {
return fmt.Errorf("could not open bind mount target: %s", err)
return errors.Wrap(err, "could not open bind mount target")
}
restrictAccess := false
var allowedNames []string
Expand All @@ -140,7 +140,7 @@ func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r *
l.Debug("hcsshim::allocateLinuxResources Hot-adding Plan9 for OCI mount")
share, err := coi.HostingSystem.AddPlan9(ctx, hostPath, uvmPathForShare, readOnly, restrictAccess, allowedNames)
if err != nil {
return fmt.Errorf("adding plan9 mount %+v: %s", mount, err)
return errors.Wrapf(err, "adding plan9 mount %+v", mount)
}
r.Add(share)
}
Expand Down
58 changes: 48 additions & 10 deletions internal/hcsoci/resources_wcow.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,12 @@ import (
"github.com/Microsoft/hcsshim/internal/uvm"
"github.com/Microsoft/hcsshim/internal/wclayer"
specs "github.com/opencontainers/runtime-spec/specs-go"
"github.com/pkg/errors"
)

func allocateWindowsResources(ctx context.Context, coi *createOptionsInternal, r *resources.Resources) error {
func allocateWindowsResources(ctx context.Context, coi *createOptionsInternal, r *resources.Resources, isSandbox bool) error {
if coi.Spec == nil || coi.Spec.Windows == nil || coi.Spec.Windows.LayerFolders == nil {
return fmt.Errorf("field 'Spec.Windows.Layerfolders' is not populated")
return errors.New("field 'Spec.Windows.Layerfolders' is not populated")
}

scratchFolder := coi.Spec.Windows.LayerFolders[len(coi.Spec.Windows.LayerFolders)-1]
Expand All @@ -32,15 +33,15 @@ func allocateWindowsResources(ctx context.Context, coi *createOptionsInternal, r
// Create the directory for the RW scratch layer if it doesn't exist
if _, err := os.Stat(scratchFolder); os.IsNotExist(err) {
if err := os.MkdirAll(scratchFolder, 0777); err != nil {
return fmt.Errorf("failed to auto-create container scratch folder %s: %s", scratchFolder, err)
return errors.Wrapf(err, "failed to auto-create container scratch folder %s", scratchFolder)
}
}

// Create sandbox.vhdx if it doesn't exist in the scratch folder. It's called sandbox.vhdx
// rather than scratch.vhdx as in the v1 schema, it's hard-coded in HCS.
if _, err := os.Stat(filepath.Join(scratchFolder, "sandbox.vhdx")); os.IsNotExist(err) {
if err := wclayer.CreateScratchLayer(ctx, scratchFolder, coi.Spec.Windows.LayerFolders[:len(coi.Spec.Windows.LayerFolders)-1]); err != nil {
return fmt.Errorf("failed to CreateSandboxLayer %s", err)
return errors.Wrap(err, "failed to CreateSandboxLayer")
}
}

Expand All @@ -53,10 +54,10 @@ func allocateWindowsResources(ctx context.Context, coi *createOptionsInternal, r
containerRootInUVM := r.ContainerRootInUVM()
containerRootPath, err := layers.MountContainerLayers(ctx, coi.Spec.Windows.LayerFolders, containerRootInUVM, coi.HostingSystem)
if err != nil {
return fmt.Errorf("failed to mount container storage: %s", err)
return errors.Wrap(err, "failed to mount container storage")
}
coi.Spec.Root.Path = containerRootPath
layers := layers.NewImageLayers(coi.HostingSystem, containerRootInUVM, coi.Spec.Windows.LayerFolders)
layers := layers.NewImageLayers(coi.HostingSystem, containerRootInUVM, coi.Spec.Windows.LayerFolders, isSandbox)
r.SetLayers(layers)
}

Expand Down Expand Up @@ -136,37 +137,74 @@ func setupMounts(ctx context.Context, coi *createOptionsInternal, r *resources.R
l.Debug("hcsshim::allocateWindowsResources Hot-adding SCSI physical disk for OCI mount")
scsiMount, err := coi.HostingSystem.AddSCSIPhysicalDisk(ctx, mount.Source, uvmPath, readOnly)
if err != nil {
return fmt.Errorf("adding SCSI physical disk mount %+v: %s", mount, err)
return errors.Wrapf(err, "adding SCSI physical disk mount %+v", mount)
}
coi.Spec.Mounts[i].Type = ""
r.Add(scsiMount)
} else if mount.Type == "virtual-disk" {
l.Debug("hcsshim::allocateWindowsResources Hot-adding SCSI virtual disk for OCI mount")
scsiMount, err := coi.HostingSystem.AddSCSI(ctx, mount.Source, uvmPath, readOnly, uvm.VMAccessTypeIndividual)
if err != nil {
return fmt.Errorf("adding SCSI virtual disk mount %+v: %s", mount, err)
return errors.Wrapf(err, "adding SCSI virtual disk mount %+v", mount)
}
coi.Spec.Mounts[i].Type = ""
r.Add(scsiMount)
} else {
if uvm.IsPipe(mount.Source) {
pipe, err := coi.HostingSystem.AddPipe(ctx, mount.Source)
if err != nil {
return fmt.Errorf("failed to add named pipe to UVM: %s", err)
return errors.Wrap(err, "failed to add named pipe to UVM")
}
r.Add(pipe)
} else {
l.Debug("hcsshim::allocateWindowsResources Hot-adding VSMB share for OCI mount")
options := coi.HostingSystem.DefaultVSMBOptions(readOnly)
share, err := coi.HostingSystem.AddVSMB(ctx, mount.Source, options)
if err != nil {
return fmt.Errorf("failed to add VSMB share to utility VM for mount %+v: %s", mount, err)
return errors.Wrapf(err, "failed to add VSMB share to utility VM for mount %+v", mount)
}
r.Add(share)
}
}
}
}

if cs, ok := coi.Spec.Windows.CredentialSpec.(string); ok {
// Only need to create a CCG instance for v2 containers
if schemaversion.IsV21(coi.actualSchemaVersion) {
hypervisorIsolated := coi.HostingSystem != nil
ccgInstance, ccgResource, err := credentials.CreateCredentialGuard(ctx, coi.actualID, cs, hypervisorIsolated)
if err != nil {
return err
}
coi.ccgState = ccgInstance.CredentialGuard
r.Add(ccgResource)
if hypervisorIsolated {
// If hypervisor isolated we need to add an hvsocket service table entry
// By default HVSocket won't allow something inside the VM to connect
// back to a process on the host. We need to update the HVSocket service table
// to allow a connection to CCG.exe on the host, so that GMSA can function.
// We need to hot add this here because at UVM creation time we don't know what containers
// will be launched in the UVM, nonetheless if they will ask for GMSA. This is a workaround
// for the previous design requirement for CCG V2 where the service entry
// must be present in the UVM'S HCS document before being sent over as hot adding
// an HvSocket service was not possible.
hvSockConfig := ccgInstance.HvSocketConfig
if err := coi.HostingSystem.UpdateHvSocketService(ctx, hvSockConfig.ServiceId, hvSockConfig.ServiceConfig); err != nil {
return errors.Wrap(err, "failed to update hvsocket service")
}
}
}
}

if coi.HostingSystem != nil && coi.hasWindowsAssignedDevices() {
windowsDevices, closers, err := handleAssignedDevicesWindows(ctx, coi.HostingSystem, coi.Spec.Annotations, coi.Spec.Windows.Devices)
if err != nil {
return err
}
r.Add(closers...)
coi.Spec.Windows.Devices = windowsDevices
}

return nil
}
24 changes: 16 additions & 8 deletions internal/layers/layers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,18 +23,26 @@ type ImageLayers struct {
vm *uvm.UtilityVM
containerRootInUVM string
layers []string
// In some instances we may want to avoid cleaning up the image layers, such as when tearing
// down a sandbox container since the UVM will be torn down shortly after and the resources
// can be cleaned up on the host.
skipCleanup bool
}

func NewImageLayers(vm *uvm.UtilityVM, containerRootInUVM string, layers []string) *ImageLayers {
func NewImageLayers(vm *uvm.UtilityVM, containerRootInUVM string, layers []string, skipCleanup bool) *ImageLayers {
return &ImageLayers{
vm: vm,
containerRootInUVM: containerRootInUVM,
layers: layers,
skipCleanup: skipCleanup,
}
}

// Release unmounts all of the layers located in the layers array.
func (layers *ImageLayers) Release(ctx context.Context, all bool) error {
if layers.skipCleanup && layers.vm != nil {
return nil
}
op := UnmountOperationSCSI
if layers.vm == nil || all {
op = UnmountOperationAll
Expand Down Expand Up @@ -244,9 +252,9 @@ func removeLCOWLayer(ctx context.Context, uvm *uvmpkg.UtilityVM, layerPath strin
}).Debug("Removed LCOW layer")
return nil
}
return fmt.Errorf("failed to remove SCSI layer: %s", err)
return errors.Wrap(err, "failed to remove SCSI layer")
}
return fmt.Errorf("failed to remove VPMEM layer: %s", err)
return errors.Wrap(err, "failed to remove VPMEM layer")
}

// UnmountOperation is used when calling Unmount() to determine what type of unmount is
Expand All @@ -270,10 +278,10 @@ func UnmountContainerLayers(ctx context.Context, layerFolders []string, containe
if uvm == nil {
// Must be an argon - folders are mounted on the host
if op != UnmountOperationAll {
return fmt.Errorf("only operation supported for host-mounted folders is unmountOperationAll")
return errors.New("only operation supported for host-mounted folders is unmountOperationAll")
}
if len(layerFolders) < 1 {
return fmt.Errorf("need at least one layer for Unmount")
return errors.New("need at least one layer for Unmount")
}
path := layerFolders[len(layerFolders)-1]
if err := wclayer.UnprepareLayer(ctx, path); err != nil {
Expand All @@ -286,7 +294,7 @@ func UnmountContainerLayers(ctx context.Context, layerFolders []string, containe

// Base+Scratch as a minimum. This is different to v1 which only requires the scratch
if len(layerFolders) < 2 {
return fmt.Errorf("at least two layers are required for unmount")
return errors.New("at least two layers are required for unmount")
}

var retError error
Expand All @@ -302,7 +310,7 @@ func UnmountContainerLayers(ctx context.Context, layerFolders []string, containe
if (op & UnmountOperationSCSI) == UnmountOperationSCSI {
hostScratchFile, err := getScratchVHDPath(layerFolders)
if err != nil {
return fmt.Errorf("failed to get scratch VHD path in layer folders: %s", err)
return errors.Wrap(err, "failed to get scratch VHD path in layer folders")
}
if err := uvm.RemoveSCSI(ctx, hostScratchFile); err != nil {
log.G(ctx).WithError(err).Warn("failed to remove scratch")
Expand Down Expand Up @@ -383,7 +391,7 @@ func getScratchVHDPath(layerFolders []string) (string, error) {
// Evaluate the symlink here (if there is one).
hostPath, err := filepath.EvalSymlinks(hostPath)
if err != nil {
return "", fmt.Errorf("failed to eval symlinks: %s", err)
return "", errors.Wrap(err, "failed to eval symlinks")
}
return hostPath, nil
}

0 comments on commit d221e81

Please sign in to comment.