Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Skip unmounting layers for sandbox container. #932

Merged
merged 1 commit into from
Jan 28, 2021
Merged

Skip unmounting layers for sandbox container. #932

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 11 additions & 10 deletions internal/hcsoci/create.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -220,7 +220,7 @@ func initializeCreateOptions(ctx context.Context, createOptions *CreateOptions)

// configureSandboxNetwork creates a new network namespace for the pod (sandbox)
// if required and then adds that namespace to the pod.
func configureSandboxNetwork(ctx context.Context, coi *createOptionsInternal, r *resources.Resources) error {
func configureSandboxNetwork(ctx context.Context, coi *createOptionsInternal, r *resources.Resources, ct oci.KubernetesContainerType) error {
if coi.NetworkNamespace != "" {
r.SetNetNS(coi.NetworkNamespace)
} else {
Expand All @@ -232,15 +232,11 @@ func configureSandboxNetwork(ctx context.Context, coi *createOptionsInternal, r
coi.actualNetworkNamespace = r.NetNS()

if coi.HostingSystem != nil {
ct, _, err := oci.GetSandboxTypeAndID(coi.Spec.Annotations)
if err != nil {
return err
}
// Only add the network namespace to a standalone or sandbox
// container but not a workload container in a sandbox that inherits
// the namespace.
if ct == oci.KubernetesContainerTypeNone || ct == oci.KubernetesContainerTypeSandbox {
if err = SetupNetworkNamespace(ctx, coi.HostingSystem, coi.actualNetworkNamespace); err != nil {
if err := SetupNetworkNamespace(ctx, coi.HostingSystem, coi.actualNetworkNamespace); err != nil {
return err
}
r.SetAddedNetNSToVM(true)
Expand Down Expand Up @@ -284,14 +280,19 @@ func CreateContainer(ctx context.Context, createOptions *CreateOptions) (_ cow.C
}
}

ct, _, err := oci.GetSandboxTypeAndID(coi.Spec.Annotations)
if err != nil {
return nil, r, err
}
isSandbox := ct == oci.KubernetesContainerTypeSandbox

// Create a network namespace if necessary.
if coi.Spec.Windows != nil &&
coi.Spec.Windows.Network != nil &&
schemaversion.IsV21(coi.actualSchemaVersion) {
err = configureSandboxNetwork(ctx, coi, r)
err = configureSandboxNetwork(ctx, coi, r, ct)
if err != nil {
return nil, r, fmt.Errorf("failure while creating namespace for container: %s", err)

}
}

Expand All @@ -302,7 +303,7 @@ func CreateContainer(ctx context.Context, createOptions *CreateOptions) (_ cow.C
return nil, r, errors.New("LCOW v1 not supported")
}
log.G(ctx).Debug("hcsshim::CreateContainer allocateLinuxResources")
err = allocateLinuxResources(ctx, coi, r)
err = allocateLinuxResources(ctx, coi, r, isSandbox)
if err != nil {
log.G(ctx).WithError(err).Debug("failed to allocateLinuxResources")
return nil, r, err
Expand All @@ -313,7 +314,7 @@ func CreateContainer(ctx context.Context, createOptions *CreateOptions) (_ cow.C
return nil, r, err
}
} else {
err = allocateWindowsResources(ctx, coi, r)
err = allocateWindowsResources(ctx, coi, r, isSandbox)
if err != nil {
log.G(ctx).WithError(err).Debug("failed to allocateWindowsResources")
return nil, r, err
Expand Down
16 changes: 8 additions & 8 deletions internal/hcsoci/resources_lcow.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ func getGPUVHDPath(coi *createOptionsInternal) (string, error) {
return gpuVHDPath, nil
}

func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r *resources.Resources) error {
func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r *resources.Resources, isSandbox bool) error {
if coi.Spec.Root == nil {
coi.Spec.Root = &specs.Root{}
}
Expand All @@ -44,10 +44,10 @@ func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r *
log.G(ctx).Debug("hcsshim::allocateLinuxResources mounting storage")
rootPath, err := layers.MountContainerLayers(ctx, coi.Spec.Windows.LayerFolders, containerRootInUVM, coi.HostingSystem)
if err != nil {
return fmt.Errorf("failed to mount container storage: %s", err)
return errors.Wrap(err, "failed to mount container storage")
}
coi.Spec.Root.Path = rootPath
layers := layers.NewImageLayers(coi.HostingSystem, containerRootInUVM, coi.Spec.Windows.LayerFolders)
layers := layers.NewImageLayers(coi.HostingSystem, containerRootInUVM, coi.Spec.Windows.LayerFolders, isSandbox)
r.SetLayers(layers)
} else if coi.Spec.Root.Path != "" {
// This is the "Plan 9" root filesystem.
Expand All @@ -56,7 +56,7 @@ func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r *
uvmPathForContainersFileSystem := path.Join(r.ContainerRootInUVM(), uvm.RootfsPath)
share, err := coi.HostingSystem.AddPlan9(ctx, hostPath, uvmPathForContainersFileSystem, coi.Spec.Root.Readonly, false, nil)
if err != nil {
return fmt.Errorf("adding plan9 root: %s", err)
return errors.Wrap(err, "adding plan9 root")
}
coi.Spec.Root.Path = uvmPathForContainersFileSystem
r.Add(share)
Expand Down Expand Up @@ -95,7 +95,7 @@ func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r *
uvmPathForShare = fmt.Sprintf(uvm.LCOWGlobalMountPrefix, coi.HostingSystem.UVMMountCounter())
scsiMount, err := coi.HostingSystem.AddSCSIPhysicalDisk(ctx, hostPath, uvmPathForShare, readOnly)
if err != nil {
return fmt.Errorf("adding SCSI physical disk mount %+v: %s", mount, err)
return errors.Wrapf(err, "adding SCSI physical disk mount %+v", mount)
}

uvmPathForFile = scsiMount.UVMPath
Expand All @@ -110,7 +110,7 @@ func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r *
// that is where it was previously mounted in UVM
scsiMount, err := coi.HostingSystem.AddSCSI(ctx, hostPath, uvmPathForShare, readOnly, uvm.VMAccessTypeIndividual)
if err != nil {
return fmt.Errorf("adding SCSI virtual disk mount %+v: %s", mount, err)
return errors.Wrapf(err, "adding SCSI virtual disk mount %+v", mount)
}

uvmPathForFile = scsiMount.UVMPath
Expand All @@ -124,7 +124,7 @@ func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r *
} else {
st, err := os.Stat(hostPath)
if err != nil {
return fmt.Errorf("could not open bind mount target: %s", err)
return errors.Wrap(err, "could not open bind mount target")
}
restrictAccess := false
var allowedNames []string
Expand All @@ -140,7 +140,7 @@ func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r *
l.Debug("hcsshim::allocateLinuxResources Hot-adding Plan9 for OCI mount")
share, err := coi.HostingSystem.AddPlan9(ctx, hostPath, uvmPathForShare, readOnly, restrictAccess, allowedNames)
if err != nil {
return fmt.Errorf("adding plan9 mount %+v: %s", mount, err)
return errors.Wrapf(err, "adding plan9 mount %+v", mount)
}
r.Add(share)
}
Expand Down
58 changes: 48 additions & 10 deletions internal/hcsoci/resources_wcow.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,12 @@ import (
"github.com/Microsoft/hcsshim/internal/uvm"
"github.com/Microsoft/hcsshim/internal/wclayer"
specs "github.com/opencontainers/runtime-spec/specs-go"
"github.com/pkg/errors"
)

func allocateWindowsResources(ctx context.Context, coi *createOptionsInternal, r *resources.Resources) error {
func allocateWindowsResources(ctx context.Context, coi *createOptionsInternal, r *resources.Resources, isSandbox bool) error {
if coi.Spec == nil || coi.Spec.Windows == nil || coi.Spec.Windows.LayerFolders == nil {
return fmt.Errorf("field 'Spec.Windows.Layerfolders' is not populated")
return errors.New("field 'Spec.Windows.Layerfolders' is not populated")
}

scratchFolder := coi.Spec.Windows.LayerFolders[len(coi.Spec.Windows.LayerFolders)-1]
Expand All @@ -32,15 +33,15 @@ func allocateWindowsResources(ctx context.Context, coi *createOptionsInternal, r
// Create the directory for the RW scratch layer if it doesn't exist
if _, err := os.Stat(scratchFolder); os.IsNotExist(err) {
if err := os.MkdirAll(scratchFolder, 0777); err != nil {
return fmt.Errorf("failed to auto-create container scratch folder %s: %s", scratchFolder, err)
return errors.Wrapf(err, "failed to auto-create container scratch folder %s", scratchFolder)
}
}

// Create sandbox.vhdx if it doesn't exist in the scratch folder. It's called sandbox.vhdx
// rather than scratch.vhdx as in the v1 schema, it's hard-coded in HCS.
if _, err := os.Stat(filepath.Join(scratchFolder, "sandbox.vhdx")); os.IsNotExist(err) {
if err := wclayer.CreateScratchLayer(ctx, scratchFolder, coi.Spec.Windows.LayerFolders[:len(coi.Spec.Windows.LayerFolders)-1]); err != nil {
return fmt.Errorf("failed to CreateSandboxLayer %s", err)
return errors.Wrap(err, "failed to CreateSandboxLayer")
}
}

Expand All @@ -53,10 +54,10 @@ func allocateWindowsResources(ctx context.Context, coi *createOptionsInternal, r
containerRootInUVM := r.ContainerRootInUVM()
containerRootPath, err := layers.MountContainerLayers(ctx, coi.Spec.Windows.LayerFolders, containerRootInUVM, coi.HostingSystem)
if err != nil {
return fmt.Errorf("failed to mount container storage: %s", err)
return errors.Wrap(err, "failed to mount container storage")
}
coi.Spec.Root.Path = containerRootPath
layers := layers.NewImageLayers(coi.HostingSystem, containerRootInUVM, coi.Spec.Windows.LayerFolders)
layers := layers.NewImageLayers(coi.HostingSystem, containerRootInUVM, coi.Spec.Windows.LayerFolders, isSandbox)
r.SetLayers(layers)
}

Expand Down Expand Up @@ -136,37 +137,74 @@ func setupMounts(ctx context.Context, coi *createOptionsInternal, r *resources.R
l.Debug("hcsshim::allocateWindowsResources Hot-adding SCSI physical disk for OCI mount")
scsiMount, err := coi.HostingSystem.AddSCSIPhysicalDisk(ctx, mount.Source, uvmPath, readOnly)
if err != nil {
return fmt.Errorf("adding SCSI physical disk mount %+v: %s", mount, err)
return errors.Wrapf(err, "adding SCSI physical disk mount %+v", mount)
}
coi.Spec.Mounts[i].Type = ""
r.Add(scsiMount)
} else if mount.Type == "virtual-disk" {
l.Debug("hcsshim::allocateWindowsResources Hot-adding SCSI virtual disk for OCI mount")
scsiMount, err := coi.HostingSystem.AddSCSI(ctx, mount.Source, uvmPath, readOnly, uvm.VMAccessTypeIndividual)
if err != nil {
return fmt.Errorf("adding SCSI virtual disk mount %+v: %s", mount, err)
return errors.Wrapf(err, "adding SCSI virtual disk mount %+v", mount)
}
coi.Spec.Mounts[i].Type = ""
r.Add(scsiMount)
} else {
if uvm.IsPipe(mount.Source) {
pipe, err := coi.HostingSystem.AddPipe(ctx, mount.Source)
if err != nil {
return fmt.Errorf("failed to add named pipe to UVM: %s", err)
return errors.Wrap(err, "failed to add named pipe to UVM")
}
r.Add(pipe)
} else {
l.Debug("hcsshim::allocateWindowsResources Hot-adding VSMB share for OCI mount")
options := coi.HostingSystem.DefaultVSMBOptions(readOnly)
share, err := coi.HostingSystem.AddVSMB(ctx, mount.Source, options)
if err != nil {
return fmt.Errorf("failed to add VSMB share to utility VM for mount %+v: %s", mount, err)
return errors.Wrapf(err, "failed to add VSMB share to utility VM for mount %+v", mount)
}
r.Add(share)
}
}
}
}

if cs, ok := coi.Spec.Windows.CredentialSpec.(string); ok {
// Only need to create a CCG instance for v2 containers
if schemaversion.IsV21(coi.actualSchemaVersion) {
hypervisorIsolated := coi.HostingSystem != nil
ccgInstance, ccgResource, err := credentials.CreateCredentialGuard(ctx, coi.actualID, cs, hypervisorIsolated)
if err != nil {
return err
}
coi.ccgState = ccgInstance.CredentialGuard
r.Add(ccgResource)
if hypervisorIsolated {
// If hypervisor isolated we need to add an hvsocket service table entry
// By default HVSocket won't allow something inside the VM to connect
// back to a process on the host. We need to update the HVSocket service table
// to allow a connection to CCG.exe on the host, so that GMSA can function.
// We need to hot add this here because at UVM creation time we don't know what containers
// will be launched in the UVM, nonetheless if they will ask for GMSA. This is a workaround
// for the previous design requirement for CCG V2 where the service entry
// must be present in the UVM'S HCS document before being sent over as hot adding
// an HvSocket service was not possible.
hvSockConfig := ccgInstance.HvSocketConfig
if err := coi.HostingSystem.UpdateHvSocketService(ctx, hvSockConfig.ServiceId, hvSockConfig.ServiceConfig); err != nil {
return errors.Wrap(err, "failed to update hvsocket service")
}
}
}
}

if coi.HostingSystem != nil && coi.hasWindowsAssignedDevices() {
windowsDevices, closers, err := handleAssignedDevicesWindows(ctx, coi.HostingSystem, coi.Spec.Annotations, coi.Spec.Windows.Devices)
if err != nil {
return err
}
r.Add(closers...)
coi.Spec.Windows.Devices = windowsDevices
}

return nil
}
24 changes: 16 additions & 8 deletions internal/layers/layers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,18 +23,26 @@ type ImageLayers struct {
vm *uvm.UtilityVM
containerRootInUVM string
layers []string
// In some instances we may want to avoid cleaning up the image layers, such as when tearing
// down a sandbox container since the UVM will be torn down shortly after and the resources
// can be cleaned up on the host.
skipCleanup bool
}

func NewImageLayers(vm *uvm.UtilityVM, containerRootInUVM string, layers []string) *ImageLayers {
func NewImageLayers(vm *uvm.UtilityVM, containerRootInUVM string, layers []string, skipCleanup bool) *ImageLayers {
return &ImageLayers{
vm: vm,
containerRootInUVM: containerRootInUVM,
layers: layers,
skipCleanup: skipCleanup,
}
}

// Release unmounts all of the layers located in the layers array.
func (layers *ImageLayers) Release(ctx context.Context, all bool) error {
if layers.skipCleanup && layers.vm != nil {
return nil
}
op := UnmountOperationSCSI
if layers.vm == nil || all {
op = UnmountOperationAll
Expand Down Expand Up @@ -244,9 +252,9 @@ func removeLCOWLayer(ctx context.Context, uvm *uvmpkg.UtilityVM, layerPath strin
}).Debug("Removed LCOW layer")
return nil
}
return fmt.Errorf("failed to remove SCSI layer: %s", err)
return errors.Wrap(err, "failed to remove SCSI layer")
}
return fmt.Errorf("failed to remove VPMEM layer: %s", err)
return errors.Wrap(err, "failed to remove VPMEM layer")
}

// UnmountOperation is used when calling Unmount() to determine what type of unmount is
Expand All @@ -270,10 +278,10 @@ func UnmountContainerLayers(ctx context.Context, layerFolders []string, containe
if uvm == nil {
// Must be an argon - folders are mounted on the host
if op != UnmountOperationAll {
return fmt.Errorf("only operation supported for host-mounted folders is unmountOperationAll")
return errors.New("only operation supported for host-mounted folders is unmountOperationAll")
}
if len(layerFolders) < 1 {
return fmt.Errorf("need at least one layer for Unmount")
return errors.New("need at least one layer for Unmount")
}
path := layerFolders[len(layerFolders)-1]
if err := wclayer.UnprepareLayer(ctx, path); err != nil {
Expand All @@ -286,7 +294,7 @@ func UnmountContainerLayers(ctx context.Context, layerFolders []string, containe

// Base+Scratch as a minimum. This is different to v1 which only requires the scratch
if len(layerFolders) < 2 {
return fmt.Errorf("at least two layers are required for unmount")
return errors.New("at least two layers are required for unmount")
}

var retError error
Expand All @@ -302,7 +310,7 @@ func UnmountContainerLayers(ctx context.Context, layerFolders []string, containe
if (op & UnmountOperationSCSI) == UnmountOperationSCSI {
hostScratchFile, err := getScratchVHDPath(layerFolders)
if err != nil {
return fmt.Errorf("failed to get scratch VHD path in layer folders: %s", err)
return errors.Wrap(err, "failed to get scratch VHD path in layer folders")
}
if err := uvm.RemoveSCSI(ctx, hostScratchFile); err != nil {
log.G(ctx).WithError(err).Warn("failed to remove scratch")
Expand Down Expand Up @@ -383,7 +391,7 @@ func getScratchVHDPath(layerFolders []string) (string, error) {
// Evaluate the symlink here (if there is one).
hostPath, err := filepath.EvalSymlinks(hostPath)
if err != nil {
return "", fmt.Errorf("failed to eval symlinks: %s", err)
return "", errors.Wrap(err, "failed to eval symlinks")
}
return hostPath, nil
}