diff --git a/Makefile b/Makefile index ac00def635..c09b1e192a 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,7 @@ clean: test: cd $(SRCROOT) && go test -v ./internal/guest/... -out/delta.tar.gz: bin/init bin/vsockexec bin/cmd/gcs bin/cmd/gcstools Makefile +out/delta.tar.gz: bin/init bin/vsockexec bin/cmd/gcs bin/cmd/gcstools bin/cmd/hooks/wait-paths Makefile @mkdir -p out rm -rf rootfs mkdir -p rootfs/bin/ @@ -40,6 +40,7 @@ out/delta.tar.gz: bin/init bin/vsockexec bin/cmd/gcs bin/cmd/gcstools Makefile cp bin/vsockexec rootfs/bin/ cp bin/cmd/gcs rootfs/bin/ cp bin/cmd/gcstools rootfs/bin/ + cp bin/cmd/hooks/wait-paths rootfs/bin/ for tool in $(GCS_TOOLS); do ln -s gcstools rootfs/bin/$$tool; done git -C $(SRCROOT) rev-parse HEAD > rootfs/gcs.commit && \ git -C $(SRCROOT) rev-parse --abbrev-ref HEAD > rootfs/gcs.branch @@ -60,6 +61,7 @@ out/initrd.img: $(BASE) out/delta.tar.gz $(SRCROOT)/hack/catcpio.sh -include deps/cmd/gcs.gomake -include deps/cmd/gcstools.gomake +-include deps/cmd/hooks/wait-paths.gomake # Implicit rule for includes that define Go targets. %.gomake: $(SRCROOT)/Makefile diff --git a/cmd/gcs/main.go b/cmd/gcs/main.go index 0926ef4d4b..13f3201fe7 100644 --- a/cmd/gcs/main.go +++ b/cmd/gcs/main.go @@ -14,20 +14,22 @@ import ( "syscall" "time" + "github.com/containerd/cgroups" + cgroupstats "github.com/containerd/cgroups/stats/v1" + oci "github.com/opencontainers/runtime-spec/specs-go" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "go.opencensus.io/trace" + "github.com/Microsoft/hcsshim/internal/guest/bridge" "github.com/Microsoft/hcsshim/internal/guest/kmsg" "github.com/Microsoft/hcsshim/internal/guest/runtime/hcsv2" "github.com/Microsoft/hcsshim/internal/guest/runtime/runc" "github.com/Microsoft/hcsshim/internal/guest/transport" + "github.com/Microsoft/hcsshim/internal/guestpath" "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/oc" "github.com/cenkalti/backoff/v4" - "github.com/containerd/cgroups" - cgroupstats "github.com/containerd/cgroups/stats/v1" - oci "github.com/opencontainers/runtime-spec/specs-go" - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "go.opencensus.io/trace" ) func memoryLogFormat(metrics *cgroupstats.Metrics) logrus.Fields { @@ -229,7 +231,7 @@ func main() { log.SetScrubbing(*scrubLogs) - baseLogPath := "/run/gcs/c" + baseLogPath := guestpath.LCOWRootPrefixInUVM logrus.Info("GCS started") diff --git a/cmd/hooks/wait-paths/main.go b/cmd/hooks/wait-paths/main.go new file mode 100644 index 0000000000..c3c49ccf9a --- /dev/null +++ b/cmd/hooks/wait-paths/main.go @@ -0,0 +1,72 @@ +// +build linux + +package main + +import ( + "context" + "fmt" + "os" + "strings" + "time" + + "github.com/sirupsen/logrus" + "github.com/urfave/cli" +) + +const ( + pathsFlag = "paths" + timeoutFlag = "timeout" +) + +// This is a hook that waits for a specific path to appear. +// The hook has required list of comma-separated paths and a default timeout in seconds. + +func main() { + app := cli.NewApp() + app.Name = "wait-paths" + app.Usage = "Provide a list paths and an optional timeout" + app.Flags = []cli.Flag{ + cli.StringFlag{ + Name: pathsFlag + ",p", + Usage: "Comma-separated list of paths that should become available", + Required: true, + }, + cli.IntFlag{ + Name: timeoutFlag + ",t", + Usage: "Timeout in seconds", + Value: 30, + }, + } + app.Action = run + if err := app.Run(os.Args); err != nil { + logrus.Fatalf("%s\n", err) + } + os.Exit(0) +} + +func run(cCtx *cli.Context) error { + timeout := cCtx.GlobalInt(timeoutFlag) + paths := strings.Split(cCtx.GlobalString(pathsFlag), ",") + + waitCtx, cancel := context.WithTimeout(context.Background(), time.Duration(timeout)*time.Second) + defer cancel() + + for _, path := range paths { + for { + if _, err := os.Stat(path); err != nil { + if !os.IsNotExist(err) { + return err + } + select { + case <-waitCtx.Done(): + return fmt.Errorf("timeout while waiting for path %q to appear", path) + default: + time.Sleep(time.Millisecond * 10) + continue + } + } + break + } + } + return nil +} diff --git a/internal/devices/drivers.go b/internal/devices/drivers.go index 4efe677e22..0dae33c963 100644 --- a/internal/devices/drivers.go +++ b/internal/devices/drivers.go @@ -8,6 +8,7 @@ import ( "fmt" "github.com/Microsoft/hcsshim/internal/cmd" + "github.com/Microsoft/hcsshim/internal/guestpath" "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/resources" "github.com/Microsoft/hcsshim/internal/uvm" @@ -45,7 +46,7 @@ func InstallKernelDriver(ctx context.Context, vm *uvm.UtilityVM, driver string) } return closer, execPnPInstallDriver(ctx, vm, uvmPath) } - uvmPathForShare := fmt.Sprintf(uvm.LCOWGlobalMountPrefix, vm.UVMMountCounter()) + uvmPathForShare := fmt.Sprintf(guestpath.LCOWGlobalMountPrefixFmt, vm.UVMMountCounter()) scsiCloser, err := vm.AddSCSI(ctx, driver, uvmPathForShare, true, false, []string{}, uvm.VMAccessTypeIndividual) if err != nil { return closer, fmt.Errorf("failed to add SCSI disk to utility VM for path %+v: %s", driver, err) diff --git a/internal/guest/runtime/hcsv2/nvidia_utils.go b/internal/guest/runtime/hcsv2/nvidia_utils.go index 84cae4718b..735bb4931d 100644 --- a/internal/guest/runtime/hcsv2/nvidia_utils.go +++ b/internal/guest/runtime/hcsv2/nvidia_utils.go @@ -10,17 +10,16 @@ import ( "os/exec" "strings" + oci "github.com/opencontainers/runtime-spec/specs-go" + "github.com/pkg/errors" + "github.com/Microsoft/hcsshim/cmd/gcstools/generichook" "github.com/Microsoft/hcsshim/internal/guest/storage/pci" + "github.com/Microsoft/hcsshim/internal/guestpath" + "github.com/Microsoft/hcsshim/internal/hooks" "github.com/Microsoft/hcsshim/pkg/annotations" - oci "github.com/opencontainers/runtime-spec/specs-go" - "github.com/pkg/errors" ) -// path that the shim mounts the nvidia gpu vhd to in the uvm -// this MUST match the path mapped to in the shim -const lcowNvidiaMountPath = "/run/nvidia" - const nvidiaDebugFilePath = "/nvidia-container.log" const nvidiaToolBinary = "nvidia-container-cli" @@ -70,35 +69,33 @@ func addNvidiaDevicePreHook(ctx context.Context, spec *oci.Spec) error { // add template for pid argument to be injected later by the generic hook binary args = append(args, "--no-cgroups", "--pid={{pid}}", spec.Root.Path) - if spec.Hooks == nil { - spec.Hooks = &oci.Hooks{} - } - hookLogDebugFileEnvOpt := fmt.Sprintf("%s=%s", generichook.LogDebugFileEnvKey, nvidiaDebugFilePath) hookEnv := append(updateEnvWithNvidiaVariables(), hookLogDebugFileEnvOpt) - nvidiaHook := oci.Hook{ - Path: genericHookPath, - Args: args, - Env: hookEnv, - } - - spec.Hooks.Prestart = append(spec.Hooks.Prestart, nvidiaHook) - return nil + nvidiaHook := hooks.NewOCIHook(genericHookPath, args, hookEnv) + return hooks.AddOCIHook(spec, hooks.Prestart, nvidiaHook) } // updateEnvWithNvidiaVariables creates an env with the nvidia gpu vhd in PATH and insecure mode set func updateEnvWithNvidiaVariables() []string { + nvidiaBin := fmt.Sprintf("%s/bin", guestpath.LCOWNvidiaMountPath) + env := updatePathEnv(nvidiaBin) + // NVC_INSECURE_MODE allows us to run nvidia-container-cli without seccomp + // we don't currently use seccomp in the uvm, so avoid using it here for now as well + env = append(env, "NVC_INSECURE_MODE=1") + return env +} + +// updatePathEnv adds specified `dirs` to PATH variable and returns the result environment variables. +func updatePathEnv(dirs ...string) []string { pathPrefix := "PATH=" - nvidiaBin := fmt.Sprintf("%s/bin", lcowNvidiaMountPath) + additionalDirs := strings.Join(dirs, ":") env := os.Environ() for i, v := range env { if strings.HasPrefix(v, pathPrefix) { - newPath := fmt.Sprintf("%s:%s", v, nvidiaBin) + newPath := fmt.Sprintf("%s:%s", v, additionalDirs) env[i] = newPath + return env } } - // NVC_INSECURE_MODE allows us to run nvidia-container-cli without seccomp - // we don't currently use seccomp in the uvm, so avoid using it here for now as well - env = append(env, "NVC_INSECURE_MODE=1") - return env + return append(env, fmt.Sprintf("PATH=%s", additionalDirs)) } diff --git a/internal/guest/runtime/hcsv2/sandbox_container.go b/internal/guest/runtime/hcsv2/sandbox_container.go index ddfbace177..c76b11d3ce 100644 --- a/internal/guest/runtime/hcsv2/sandbox_container.go +++ b/internal/guest/runtime/hcsv2/sandbox_container.go @@ -10,16 +10,18 @@ import ( "path/filepath" "strings" - "github.com/Microsoft/hcsshim/internal/guest/network" - "github.com/Microsoft/hcsshim/internal/oc" - "github.com/Microsoft/hcsshim/pkg/annotations" oci "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" "go.opencensus.io/trace" + + "github.com/Microsoft/hcsshim/internal/guest/network" + "github.com/Microsoft/hcsshim/internal/guestpath" + "github.com/Microsoft/hcsshim/internal/oc" + "github.com/Microsoft/hcsshim/pkg/annotations" ) func getSandboxRootDir(id string) string { - return filepath.Join("/run/gcs/c", id) + return filepath.Join(guestpath.LCOWRootPrefixInUVM, id) } func getSandboxHugePageMountsDir(id string) string { diff --git a/internal/guest/runtime/hcsv2/spec.go b/internal/guest/runtime/hcsv2/spec.go index a0be478804..5241ba6431 100644 --- a/internal/guest/runtime/hcsv2/spec.go +++ b/internal/guest/runtime/hcsv2/spec.go @@ -10,12 +10,14 @@ import ( "strconv" "strings" - "github.com/Microsoft/hcsshim/internal/log" - "github.com/Microsoft/hcsshim/pkg/annotations" "github.com/opencontainers/runc/libcontainer/devices" "github.com/opencontainers/runc/libcontainer/user" oci "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" + + "github.com/Microsoft/hcsshim/internal/hooks" + "github.com/Microsoft/hcsshim/internal/log" + "github.com/Microsoft/hcsshim/pkg/annotations" ) // getNetworkNamespaceID returns the `ToLower` of @@ -257,17 +259,7 @@ func applyAnnotationsToSpec(ctx context.Context, spec *oci.Spec) error { } // Helper function to create an oci prestart hook to run ldconfig -func addLDConfigHook(ctx context.Context, spec *oci.Spec, args, env []string) error { - if spec.Hooks == nil { - spec.Hooks = &oci.Hooks{} - } - - ldConfigHook := oci.Hook{ - Path: "/sbin/ldconfig", - Args: args, - Env: env, - } - - spec.Hooks.Prestart = append(spec.Hooks.Prestart, ldConfigHook) - return nil +func addLDConfigHook(_ context.Context, spec *oci.Spec, args, env []string) error { + ldConfigHook := hooks.NewOCIHook("/sbin/ldconfig", args, env) + return hooks.AddOCIHook(spec, hooks.Prestart, ldConfigHook) } diff --git a/internal/guest/runtime/hcsv2/standalone_container.go b/internal/guest/runtime/hcsv2/standalone_container.go index 8ea0275cac..89189313e8 100644 --- a/internal/guest/runtime/hcsv2/standalone_container.go +++ b/internal/guest/runtime/hcsv2/standalone_container.go @@ -10,15 +10,17 @@ import ( "path/filepath" "strings" - "github.com/Microsoft/hcsshim/internal/guest/network" - "github.com/Microsoft/hcsshim/internal/oc" oci "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" "go.opencensus.io/trace" + + "github.com/Microsoft/hcsshim/internal/guest/network" + "github.com/Microsoft/hcsshim/internal/guestpath" + "github.com/Microsoft/hcsshim/internal/oc" ) func getStandaloneRootDir(id string) string { - return filepath.Join("/run/gcs/c", id) + return filepath.Join(guestpath.LCOWRootPrefixInUVM, id) } func getStandaloneHostnamePath(id string) string { diff --git a/internal/guest/runtime/hcsv2/uvm.go b/internal/guest/runtime/hcsv2/uvm.go index 36e9834110..d5e004fc6a 100644 --- a/internal/guest/runtime/hcsv2/uvm.go +++ b/internal/guest/runtime/hcsv2/uvm.go @@ -225,12 +225,16 @@ func (h *Host) CreateContainer(ctx context.Context, id string, settings *prot.VM // We append the variable after the security policy enforcing logic completes so as to bypass it; the // security policy variable cannot be included in the security policy as its value is not available // security policy construction time. - if policyEnforcer, ok := (h.securityPolicyEnforcer).(*securitypolicy.StandardSecurityPolicyEnforcer); ok { secPolicyEnv := fmt.Sprintf("SECURITY_POLICY=%s", policyEnforcer.EncodedSecurityPolicy) settings.OCISpecification.Process.Env = append(settings.OCISpecification.Process.Env, secPolicyEnv) } + // Sandbox mount paths need to be resolved in the spec before expected mounts policy can be enforced. + if err = h.securityPolicyEnforcer.EnforceExpectedMountsPolicy(id, settings.OCISpecification); err != nil { + return nil, errors.Wrapf(err, "container creation denied due to policy") + } + // Create the BundlePath if err := os.MkdirAll(settings.OCIBundlePath, 0700); err != nil { return nil, errors.Wrapf(err, "failed to create OCIBundlePath: '%s'", settings.OCIBundlePath) diff --git a/internal/guest/runtime/hcsv2/workload_container.go b/internal/guest/runtime/hcsv2/workload_container.go index d271c162ef..b9679d9036 100644 --- a/internal/guest/runtime/hcsv2/workload_container.go +++ b/internal/guest/runtime/hcsv2/workload_container.go @@ -9,16 +9,18 @@ import ( "path/filepath" "strings" - "github.com/Microsoft/hcsshim/internal/oc" - "github.com/Microsoft/hcsshim/pkg/annotations" oci "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" "go.opencensus.io/trace" "golang.org/x/sys/unix" + + "github.com/Microsoft/hcsshim/internal/guestpath" + "github.com/Microsoft/hcsshim/internal/oc" + "github.com/Microsoft/hcsshim/pkg/annotations" ) func getWorkloadRootDir(id string) string { - return filepath.Join("/run/gcs/c", id) + return filepath.Join(guestpath.LCOWRootPrefixInUVM, id) } // os.MkdirAll combines the given permissions with the running process's @@ -32,11 +34,10 @@ func mkdirAllModePerm(target string) error { } func updateSandboxMounts(sbid string, spec *oci.Spec) error { - sandboxMountPrefix := "sandbox://" for i, m := range spec.Mounts { - if strings.HasPrefix(m.Source, sandboxMountPrefix) { + if strings.HasPrefix(m.Source, guestpath.SandboxMountPrefix) { mountsDir := getSandboxMountsDir(sbid) - subPath := strings.TrimPrefix(m.Source, sandboxMountPrefix) + subPath := strings.TrimPrefix(m.Source, guestpath.SandboxMountPrefix) sandboxSource := filepath.Join(mountsDir, subPath) // filepath.Join cleans the resulting path before returning so it would resolve the relative path if one was given. @@ -59,11 +60,10 @@ func updateSandboxMounts(sbid string, spec *oci.Spec) error { } func updateHugePageMounts(sbid string, spec *oci.Spec) error { - mountPrefix := "hugepages://" for i, m := range spec.Mounts { - if strings.HasPrefix(m.Source, mountPrefix) { + if strings.HasPrefix(m.Source, guestpath.HugePagesMountPrefix) { mountsDir := getSandboxHugePageMountsDir(sbid) - subPath := strings.TrimPrefix(m.Source, mountPrefix) + subPath := strings.TrimPrefix(m.Source, guestpath.HugePagesMountPrefix) pageSize := strings.Split(subPath, string(os.PathSeparator))[0] hugePageMountSource := filepath.Join(mountsDir, subPath) diff --git a/internal/guest/storage/test/policy/mountmonitoringsecuritypolicyenforcer.go b/internal/guest/storage/test/policy/mountmonitoringsecuritypolicyenforcer.go index bb06d465a8..aa34077fcb 100644 --- a/internal/guest/storage/test/policy/mountmonitoringsecuritypolicyenforcer.go +++ b/internal/guest/storage/test/policy/mountmonitoringsecuritypolicyenforcer.go @@ -1,6 +1,8 @@ package policy import ( + oci "github.com/opencontainers/runtime-spec/specs-go" + "github.com/Microsoft/hcsshim/pkg/securitypolicy" ) @@ -32,3 +34,7 @@ func (p *MountMonitoringSecurityPolicyEnforcer) EnforceOverlayMountPolicy(contai func (p *MountMonitoringSecurityPolicyEnforcer) EnforceCreateContainerPolicy(_ string, _ []string, _ []string, _ string) (err error) { return nil } + +func (p *MountMonitoringSecurityPolicyEnforcer) EnforceExpectedMountsPolicy(_ string, _ *oci.Spec) error { + return nil +} diff --git a/internal/guestpath/paths.go b/internal/guestpath/paths.go new file mode 100644 index 0000000000..62bbfeb636 --- /dev/null +++ b/internal/guestpath/paths.go @@ -0,0 +1,24 @@ +package guestpath + +const ( + // LCOWNvidiaMountPath is the path format in LCOW UVM where nvidia tools are mounted + // keep this value in sync with opengcs + LCOWNvidiaMountPath = "/run/nvidia" + // LCOWRootPrefixInUVM is the path inside UVM where LCOW container's root file system will be mounted + LCOWRootPrefixInUVM = "/run/gcs/c" + // WCOWRootPrefixInUVM is the path inside UVM where WCOW container's root file system will be mounted + WCOWRootPrefixInUVM = `C:\c` + // SandboxMountPrefix is mount prefix used in container spec to mark a sandbox-mount + SandboxMountPrefix = "sandbox://" + // HugePagesMountPrefix is mount prefix used in container spec to mark a huge-pages mount + HugePagesMountPrefix = "hugepages://" + // LCOWMountPathPrefixFmt is the path format in the LCOW UVM where non global mounts, such + // as Plan9 mounts are added + LCOWMountPathPrefixFmt = "/mounts/m%d" + // LCOWGlobalMountPrefixFmt is the path format in the LCOW UVM where global mounts are added + LCOWGlobalMountPrefixFmt = "/run/mounts/m%d" + // WCOWGlobalMountPrefixFmt is the path prefix format in the WCOW UVM where mounts are added + WCOWGlobalMountPrefixFmt = "C:\\mounts\\m%d" + // RootfsPath is part of the container's rootfs path + RootfsPath = "rootfs" +) diff --git a/internal/hcsoci/create.go b/internal/hcsoci/create.go index 16821a1861..058530aac1 100644 --- a/internal/hcsoci/create.go +++ b/internal/hcsoci/create.go @@ -14,6 +14,7 @@ import ( "github.com/Microsoft/go-winio/pkg/guid" "github.com/Microsoft/hcsshim/internal/clone" "github.com/Microsoft/hcsshim/internal/cow" + "github.com/Microsoft/hcsshim/internal/guestpath" "github.com/Microsoft/hcsshim/internal/hcs" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" "github.com/Microsoft/hcsshim/internal/log" @@ -27,8 +28,8 @@ import ( ) var ( - lcowRootInUVM = "/run/gcs/c/%s" - wcowRootInUVM = `C:\c\%s` + lcowRootInUVM = guestpath.LCOWRootPrefixInUVM + "/%s" + wcowRootInUVM = guestpath.WCOWRootPrefixInUVM + "/%s" ) // CreateOptions are the set of fields used to call CreateContainer(). diff --git a/internal/hcsoci/devices.go b/internal/hcsoci/devices.go index 42d11aac52..4f17a715d6 100644 --- a/internal/hcsoci/devices.go +++ b/internal/hcsoci/devices.go @@ -9,7 +9,11 @@ import ( "path/filepath" "strconv" + specs "github.com/opencontainers/runtime-spec/specs-go" + "github.com/pkg/errors" + "github.com/Microsoft/hcsshim/internal/devices" + "github.com/Microsoft/hcsshim/internal/guestpath" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/oci" @@ -17,8 +21,6 @@ import ( "github.com/Microsoft/hcsshim/internal/uvm" "github.com/Microsoft/hcsshim/osversion" "github.com/Microsoft/hcsshim/pkg/annotations" - specs "github.com/opencontainers/runtime-spec/specs-go" - "github.com/pkg/errors" ) const deviceUtilExeName = "device-util.exe" @@ -222,14 +224,14 @@ func handleAssignedDevicesLCOW( scsiMount, err := vm.AddSCSI( ctx, gpuSupportVhdPath, - uvm.LCOWNvidiaMountPath, + guestpath.LCOWNvidiaMountPath, true, false, options, uvm.VMAccessTypeNoop, ) if err != nil { - return resultDevs, closers, errors.Wrapf(err, "failed to add scsi device %s in the UVM %s at %s", gpuSupportVhdPath, vm.ID(), uvm.LCOWNvidiaMountPath) + return resultDevs, closers, errors.Wrapf(err, "failed to add scsi device %s in the UVM %s at %s", gpuSupportVhdPath, vm.ID(), guestpath.LCOWNvidiaMountPath) } closers = append(closers, scsiMount) } diff --git a/internal/hcsoci/hcsdoc_wcow.go b/internal/hcsoci/hcsdoc_wcow.go index 688d901609..b3080399a6 100644 --- a/internal/hcsoci/hcsdoc_wcow.go +++ b/internal/hcsoci/hcsdoc_wcow.go @@ -11,6 +11,10 @@ import ( "regexp" "strings" + specs "github.com/opencontainers/runtime-spec/specs-go" + "github.com/sirupsen/logrus" + + "github.com/Microsoft/hcsshim/internal/guestpath" "github.com/Microsoft/hcsshim/internal/hcs/schema1" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" "github.com/Microsoft/hcsshim/internal/layers" @@ -22,8 +26,6 @@ import ( "github.com/Microsoft/hcsshim/internal/wclayer" "github.com/Microsoft/hcsshim/osversion" "github.com/Microsoft/hcsshim/pkg/annotations" - specs "github.com/opencontainers/runtime-spec/specs-go" - "github.com/sirupsen/logrus" ) // A simple wrapper struct around the container mount configs that should be added to the @@ -78,7 +80,7 @@ func createMountsConfig(ctx context.Context, coi *createOptionsInternal) (*mount return nil, err } mdv2.HostPath = uvmPath - } else if strings.HasPrefix(mount.Source, "sandbox://") { + } else if strings.HasPrefix(mount.Source, guestpath.SandboxMountPrefix) { // Convert to the path in the guest that was asked for. mdv2.HostPath = convertToWCOWSandboxMountPath(mount.Source) } else { diff --git a/internal/hcsoci/resources_lcow.go b/internal/hcsoci/resources_lcow.go index 085854a4c7..d438738c28 100644 --- a/internal/hcsoci/resources_lcow.go +++ b/internal/hcsoci/resources_lcow.go @@ -13,12 +13,14 @@ import ( "path/filepath" "strings" + specs "github.com/opencontainers/runtime-spec/specs-go" + "github.com/pkg/errors" + + "github.com/Microsoft/hcsshim/internal/guestpath" "github.com/Microsoft/hcsshim/internal/layers" "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/resources" "github.com/Microsoft/hcsshim/internal/uvm" - specs "github.com/opencontainers/runtime-spec/specs-go" - "github.com/pkg/errors" ) func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r *resources.Resources, isSandbox bool) error { @@ -39,7 +41,7 @@ func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r * // This is the "Plan 9" root filesystem. // TODO: We need a test for this. Ask @jstarks how you can even lay this out on Windows. hostPath := coi.Spec.Root.Path - uvmPathForContainersFileSystem := path.Join(r.ContainerRootInUVM(), uvm.RootfsPath) + uvmPathForContainersFileSystem := path.Join(r.ContainerRootInUVM(), guestpath.RootfsPath) share, err := coi.HostingSystem.AddPlan9(ctx, hostPath, uvmPathForContainersFileSystem, coi.Spec.Root.Readonly, false, nil) if err != nil { return errors.Wrap(err, "adding plan9 root") @@ -65,7 +67,7 @@ func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r * if coi.HostingSystem != nil { hostPath := mount.Source - uvmPathForShare := path.Join(containerRootInUVM, fmt.Sprintf(uvm.LCOWMountPathPrefix, i)) + uvmPathForShare := path.Join(containerRootInUVM, fmt.Sprintf(guestpath.LCOWMountPathPrefixFmt, i)) uvmPathForFile := uvmPathForShare readOnly := false @@ -79,7 +81,7 @@ func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r * l := log.G(ctx).WithField("mount", fmt.Sprintf("%+v", mount)) if mount.Type == "physical-disk" { l.Debug("hcsshim::allocateLinuxResources Hot-adding SCSI physical disk for OCI mount") - uvmPathForShare = fmt.Sprintf(uvm.LCOWGlobalMountPrefix, coi.HostingSystem.UVMMountCounter()) + uvmPathForShare = fmt.Sprintf(guestpath.LCOWGlobalMountPrefixFmt, coi.HostingSystem.UVMMountCounter()) scsiMount, err := coi.HostingSystem.AddSCSIPhysicalDisk(ctx, hostPath, uvmPathForShare, readOnly, mount.Options) if err != nil { return errors.Wrapf(err, "adding SCSI physical disk mount %+v", mount) @@ -90,7 +92,7 @@ func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r * coi.Spec.Mounts[i].Type = "none" } else if mount.Type == "virtual-disk" { l.Debug("hcsshim::allocateLinuxResources Hot-adding SCSI virtual disk for OCI mount") - uvmPathForShare = fmt.Sprintf(uvm.LCOWGlobalMountPrefix, coi.HostingSystem.UVMMountCounter()) + uvmPathForShare = fmt.Sprintf(guestpath.LCOWGlobalMountPrefixFmt, coi.HostingSystem.UVMMountCounter()) // if the scsi device is already attached then we take the uvm path that the function below returns // that is where it was previously mounted in UVM @@ -110,15 +112,19 @@ func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r * uvmPathForFile = scsiMount.UVMPath r.Add(scsiMount) coi.Spec.Mounts[i].Type = "none" - } else if strings.HasPrefix(mount.Source, "sandbox://") { + } else if strings.HasPrefix(mount.Source, guestpath.SandboxMountPrefix) { // Mounts that map to a path in UVM are specified with 'sandbox://' prefix. // example: sandbox:///a/dirInUvm destination:/b/dirInContainer uvmPathForFile = mount.Source - } else if strings.HasPrefix(mount.Source, "hugepages://") { + } else if strings.HasPrefix(mount.Source, guestpath.HugePagesMountPrefix) { // currently we only support 2M hugepage size - hugePageSubDirs := strings.Split(strings.TrimPrefix(mount.Source, "hugepages://"), "/") + hugePageSubDirs := strings.Split(strings.TrimPrefix(mount.Source, guestpath.HugePagesMountPrefix), "/") if len(hugePageSubDirs) < 2 { - return errors.Errorf(`%s mount path is invalid, expected format: hugepages:///`, mount.Source) + return errors.Errorf( + `%s mount path is invalid, expected format: %s/`, + mount.Source, + guestpath.HugePagesMountPrefix, + ) } // hugepages:// should be followed by pagesize @@ -155,15 +161,17 @@ func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r * } } - if coi.HostingSystem != nil { - if coi.hasWindowsAssignedDevices() { - windowsDevices, closers, err := handleAssignedDevicesLCOW(ctx, coi.HostingSystem, coi.Spec.Annotations, coi.Spec.Windows.Devices) - if err != nil { - return err - } - r.Add(closers...) - coi.Spec.Windows.Devices = windowsDevices + if coi.HostingSystem == nil { + return nil + } + + if coi.hasWindowsAssignedDevices() { + windowsDevices, closers, err := handleAssignedDevicesLCOW(ctx, coi.HostingSystem, coi.Spec.Annotations, coi.Spec.Windows.Devices) + if err != nil { + return err } + r.Add(closers...) + coi.Spec.Windows.Devices = windowsDevices } return nil } diff --git a/internal/hcsoci/resources_wcow.go b/internal/hcsoci/resources_wcow.go index 1150ca5c3c..fa22c8047e 100644 --- a/internal/hcsoci/resources_wcow.go +++ b/internal/hcsoci/resources_wcow.go @@ -13,16 +13,18 @@ import ( "path/filepath" "strings" + specs "github.com/opencontainers/runtime-spec/specs-go" + "github.com/pkg/errors" + "github.com/Microsoft/hcsshim/internal/cmd" "github.com/Microsoft/hcsshim/internal/credentials" + "github.com/Microsoft/hcsshim/internal/guestpath" "github.com/Microsoft/hcsshim/internal/layers" "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/resources" "github.com/Microsoft/hcsshim/internal/schemaversion" "github.com/Microsoft/hcsshim/internal/uvm" "github.com/Microsoft/hcsshim/internal/wclayer" - specs "github.com/opencontainers/runtime-spec/specs-go" - "github.com/pkg/errors" ) const wcowSandboxMountPath = "C:\\SandboxMounts" @@ -140,7 +142,7 @@ func setupMounts(ctx context.Context, coi *createOptionsInternal, r *resources.R } if coi.HostingSystem != nil && schemaversion.IsV21(coi.actualSchemaVersion) { - uvmPath := fmt.Sprintf(uvm.WCOWGlobalMountPrefix, coi.HostingSystem.UVMMountCounter()) + uvmPath := fmt.Sprintf(guestpath.WCOWGlobalMountPrefixFmt, coi.HostingSystem.UVMMountCounter()) readOnly := false for _, o := range mount.Options { if strings.ToLower(o) == "ro" { @@ -178,7 +180,7 @@ func setupMounts(ctx context.Context, coi *createOptionsInternal, r *resources.R return errors.Wrapf(err, "adding SCSI EVD mount failed %+v", mount) } r.Add(scsiMount) - } else if strings.HasPrefix(mount.Source, "sandbox://") { + } else if strings.HasPrefix(mount.Source, guestpath.SandboxMountPrefix) { // Mounts that map to a path in the UVM are specified with a 'sandbox://' prefix. // // Example: sandbox:///a/dirInUvm destination:C:\\dirInContainer. @@ -228,6 +230,6 @@ func setupMounts(ctx context.Context, coi *createOptionsInternal, r *resources.R } func convertToWCOWSandboxMountPath(source string) string { - subPath := strings.TrimPrefix(source, "sandbox://") + subPath := strings.TrimPrefix(source, guestpath.SandboxMountPrefix) return filepath.Join(wcowSandboxMountPath, subPath) } diff --git a/internal/hooks/spec.go b/internal/hooks/spec.go new file mode 100644 index 0000000000..51ba3aa592 --- /dev/null +++ b/internal/hooks/spec.go @@ -0,0 +1,53 @@ +package hooks + +import ( + "fmt" + + oci "github.com/opencontainers/runtime-spec/specs-go" +) + +// Note: The below type definition as well as constants have been copied from +// https://github.com/opencontainers/runc/blob/master/libcontainer/configs/config.go. +// This is done to not introduce a direct dependency on runc, which would complicate +// integration with windows. +type HookName string + +const ( + + // Prestart commands are executed after the container namespaces are created, + // but before the user supplied command is executed from init. + // Note: This hook is now deprecated + // Prestart commands are called in the Runtime namespace. + Prestart HookName = "prestart" + + // CreateRuntime commands MUST be called as part of the create operation after + // the runtime environment has been created but before the pivot_root has been executed. + // CreateRuntime is called immediately after the deprecated Prestart hook. + // CreateRuntime commands are called in the Runtime Namespace. + CreateRuntime HookName = "createRuntime" +) + +// NewOCIHook creates a new oci.Hook with given parameters +func NewOCIHook(path string, args, env []string) oci.Hook { + return oci.Hook{ + Path: path, + Args: args, + Env: env, + } +} + +// AddOCIHook adds oci.Hook of the given hook name to spec +func AddOCIHook(spec *oci.Spec, hn HookName, hk oci.Hook) error { + if spec.Hooks == nil { + spec.Hooks = &oci.Hooks{} + } + switch hn { + case Prestart: + spec.Hooks.Prestart = append(spec.Hooks.Prestart, hk) + case CreateRuntime: + spec.Hooks.CreateRuntime = append(spec.Hooks.CreateRuntime, hk) + default: + return fmt.Errorf("hook %q is not supported", hn) + } + return nil +} diff --git a/internal/layers/layers.go b/internal/layers/layers.go index 573ad72aa7..0334076365 100644 --- a/internal/layers/layers.go +++ b/internal/layers/layers.go @@ -11,15 +11,17 @@ import ( "path/filepath" "time" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "golang.org/x/sys/windows" + + "github.com/Microsoft/hcsshim/internal/guestpath" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" "github.com/Microsoft/hcsshim/internal/hcserror" "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/ospath" "github.com/Microsoft/hcsshim/internal/uvm" "github.com/Microsoft/hcsshim/internal/wclayer" - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "golang.org/x/sys/windows" ) // ImageLayers contains all the layers for an image. @@ -261,7 +263,7 @@ func MountContainerLayers(ctx context.Context, containerID string, layerFolders err = vm.CombineLayersWCOW(ctx, layers, containerScratchPathInUVM) rootfs = containerScratchPathInUVM } else { - rootfs = ospath.Join(vm.OS(), guestRoot, uvm.RootfsPath) + rootfs = ospath.Join(vm.OS(), guestRoot, guestpath.RootfsPath) err = vm.CombineLayersLCOW(ctx, containerID, lcowUvmLayerPaths, containerScratchPathInUVM, rootfs) } if err != nil { @@ -289,7 +291,7 @@ func addLCOWLayer(ctx context.Context, vm *uvm.UtilityVM, layerPath string) (uvm } options := []string{"ro"} - uvmPath = fmt.Sprintf(uvm.LCOWGlobalMountPrefix, vm.UVMMountCounter()) + uvmPath = fmt.Sprintf(guestpath.LCOWGlobalMountPrefixFmt, vm.UVMMountCounter()) sm, err := vm.AddSCSI(ctx, layerPath, uvmPath, true, false, options, uvm.VMAccessTypeNoop) if err != nil { return "", fmt.Errorf("failed to add SCSI layer: %s", err) @@ -460,7 +462,7 @@ func containerRootfsPath(vm *uvm.UtilityVM, rootPath string) string { if vm.OS() == "windows" { return ospath.Join(vm.OS(), rootPath) } - return ospath.Join(vm.OS(), rootPath, uvm.RootfsPath) + return ospath.Join(vm.OS(), rootPath, guestpath.RootfsPath) } func getScratchVHDPath(layerFolders []string) (string, error) { diff --git a/internal/tools/securitypolicy/README.md b/internal/tools/securitypolicy/README.md index f15ab873e0..4efb7c1632 100644 --- a/internal/tools/securitypolicy/README.md +++ b/internal/tools/securitypolicy/README.md @@ -21,6 +21,7 @@ be downloaded, turned into an ext4, and finally a dm-verity root hash calculated image_name = "rust:1.52.1" command = ["rustc", "--help"] working_dir = "/home/user" +expected_mounts = ["/path/to/container/mount-1", "/path/to/container/mount-2"] [[container.env_rule]] strategy = "re2" @@ -86,7 +87,14 @@ represented in JSON. "5": "1b80f120dbd88e4355d6241b519c3e25290215c469516b49dece9cf07175a766" } }, - "working_dir": "/home/user" + "working_dir": "/home/user", + "expected_mounts": { + "length": 2, + "elements": { + "0": "/path/to/container/mount-1", + "1": "/path/to/container/mount-2" + } + } }, "1": { "command": { @@ -114,7 +122,11 @@ represented in JSON. "0": "16b514057a06ad665f92c02863aca074fd5976c755d26bff16365299169e8415" } }, - "working_dir": "/" + "working_dir": "/", + "expected_mounts": { + "length": 0, + "elements": {} + } } } } @@ -135,11 +147,11 @@ output raw JSON in addition to the Base64 encoded version Some images will be pulled from registries that require authorization. To add authorization information for a given image, you would add an `[auth]` object -to the TOML definiton for that image. For example: +to the TOML definition for that image. For example: ```toml -[[image]] -image_name = "rust:1.52.1" +[[container]] +name = "rust:1.52.1" command = ["rustc", "--help"] [auth] @@ -147,8 +159,8 @@ username = "my username" password = "my password" ``` -Authorization information needs added on a per-image basis as it can vary from -image to image and their respective registries. +Authorization information needs to be added on a per-image basis as it can vary +from image to image and their respective registries. To pull an image using anonymous access, no `[auth]` object is required. diff --git a/internal/tools/securitypolicy/helpers/helpers.go b/internal/tools/securitypolicy/helpers/helpers.go index 614e5c10e8..103b2bae3a 100644 --- a/internal/tools/securitypolicy/helpers/helpers.go +++ b/internal/tools/securitypolicy/helpers/helpers.go @@ -74,6 +74,7 @@ func DefaultContainerConfigs() []securitypolicy.ContainerConfig { []securitypolicy.EnvRuleConfig{}, securitypolicy.AuthConfig{}, "", + []string{}, ) return []securitypolicy.ContainerConfig{pause} } @@ -136,7 +137,13 @@ func PolicyContainersFromConfigs(containerConfigs []securitypolicy.ContainerConf workingDir = containerConfig.WorkingDir } - container, err := securitypolicy.NewContainer(containerConfig.Command, layerHashes, envRules, workingDir) + container, err := securitypolicy.NewContainer( + containerConfig.Command, + layerHashes, + envRules, + workingDir, + containerConfig.ExpectedMounts, + ) if err != nil { return nil, err } diff --git a/internal/tools/securitypolicy/main.go b/internal/tools/securitypolicy/main.go index 4285541ea6..4073a5d847 100644 --- a/internal/tools/securitypolicy/main.go +++ b/internal/tools/securitypolicy/main.go @@ -75,7 +75,6 @@ func createPolicyFromConfig(config *securitypolicy.PolicyConfig) (*securitypolic // and any environment variable rules we might need defaultContainers := helpers.DefaultContainerConfigs() config.Containers = append(config.Containers, defaultContainers...) - policyContainers, err := helpers.PolicyContainersFromConfigs(config.Containers) if err != nil { return nil, err diff --git a/internal/uvm/constants.go b/internal/uvm/constants.go index 842a8a5e8c..1ddcf903ab 100644 --- a/internal/uvm/constants.go +++ b/internal/uvm/constants.go @@ -16,19 +16,6 @@ const ( // DefaultVPMemSizeBytes is the default size of a VPMem device if the create request // doesn't specify. DefaultVPMemSizeBytes = 4 * 1024 * 1024 * 1024 // 4GB - - // LCOWMountPathPrefix is the path format in the LCOW UVM where non global mounts, such - // as Plan9 mounts are added - LCOWMountPathPrefix = "/mounts/m%d" - // LCOWGlobalMountPrefix is the path format in the LCOW UVM where global mounts are added - LCOWGlobalMountPrefix = "/run/mounts/m%d" - // LCOWNvidiaMountPath is the path format in LCOW UVM where nvidia tools are mounted - // keep this value in sync with opengcs - LCOWNvidiaMountPath = "/run/nvidia" - // WCOWGlobalMountPrefix is the path prefix format in the WCOW UVM where mounts are added - WCOWGlobalMountPrefix = "C:\\mounts\\m%d" - // RootfsPath is part of the container's rootfs path - RootfsPath = "rootfs" ) var ( diff --git a/pkg/annotations/annotations.go b/pkg/annotations/annotations.go index 7a51b2a0e4..2f06dfb8bd 100644 --- a/pkg/annotations/annotations.go +++ b/pkg/annotations/annotations.go @@ -261,7 +261,7 @@ const ( // GuestStateFile specifies the path of the vmgs file to use if required. Only applies in SNP mode. GuestStateFile = "io.microsoft.virtualmachine.lcow.gueststatefile" - // AnnotationDisableLCOWTimeSyncService is used to disable the chronyd time + // DisableLCOWTimeSyncService is used to disable the chronyd time // synchronization service inside the LCOW UVM. DisableLCOWTimeSyncService = "io.microsoft.virtualmachine.lcow.timesync.disable" diff --git a/pkg/securitypolicy/securitypolicy.go b/pkg/securitypolicy/securitypolicy.go index 87cf644aef..2a539ed9db 100644 --- a/pkg/securitypolicy/securitypolicy.go +++ b/pkg/securitypolicy/securitypolicy.go @@ -38,11 +38,12 @@ type EnvRuleConfig struct { // ContainerConfig contains toml or JSON config for container described // in security policy. type ContainerConfig struct { - ImageName string `json:"image_name" toml:"image_name"` - Command []string `json:"command" toml:"command"` - Auth AuthConfig `json:"auth" toml:"auth"` - EnvRules []EnvRuleConfig `json:"env_rules" toml:"env_rule"` - WorkingDir string `json:"working_dir" toml:"working_dir"` + ImageName string `json:"image_name" toml:"image_name"` + Command []string `json:"command" toml:"command"` + Auth AuthConfig `json:"auth" toml:"auth"` + EnvRules []EnvRuleConfig `json:"env_rules" toml:"env_rule"` + WorkingDir string `json:"working_dir" toml:"working_dir"` + ExpectedMounts []string `json:"expected_mounts" toml:"expected_mounts"` } // NewContainerConfig creates a new ContainerConfig from the given values. @@ -52,13 +53,15 @@ func NewContainerConfig( envRules []EnvRuleConfig, auth AuthConfig, workingDir string, + expectedMounts []string, ) ContainerConfig { return ContainerConfig{ - ImageName: imageName, - Command: command, - EnvRules: envRules, - Auth: auth, - WorkingDir: workingDir, + ImageName: imageName, + Command: command, + EnvRules: envRules, + Auth: auth, + WorkingDir: workingDir, + ExpectedMounts: expectedMounts, } } @@ -97,6 +100,9 @@ type securityPolicyContainer struct { // WorkingDir is a path to container's working directory, which all the processes // will default to. WorkingDir string + // Unordered list of mounts which are expected to be present when the container + // starts + ExpectedMounts []string `json:"expected_mounts"` } // SecurityPolicyState is a structure that holds user supplied policy to enforce @@ -119,7 +125,7 @@ type EncodedSecurityPolicy struct { // security policy for given policy. The security policy is transmitted as json // in an annotation, so we first have to remove the base64 encoding that allows // the JSON based policy to be passed as a string. From there, we decode the -// JSONand setup our security policy struct +// JSON and setup our security policy struct func NewSecurityPolicyState(base64Policy string) (*SecurityPolicyState, error) { // construct an encoded security policy that holds the base64 representation encodedSecurityPolicy := EncodedSecurityPolicy{ @@ -175,10 +181,11 @@ type Containers struct { } type Container struct { - Command CommandArgs `json:"command"` - EnvRules EnvRules `json:"env_rules"` - Layers Layers `json:"layers"` - WorkingDir string `json:"working_dir"` + Command CommandArgs `json:"command"` + EnvRules EnvRules `json:"env_rules"` + Layers Layers `json:"layers"` + WorkingDir string `json:"working_dir"` + ExpectedMounts ExpectedMounts `json:"expected_mounts"` } type Layers struct { @@ -198,17 +205,23 @@ type EnvRules struct { Elements map[string]EnvRuleConfig `json:"elements"` } +type ExpectedMounts struct { + Length int `json:"length"` + Elements map[string]string `json:"elements"` +} + // NewContainer creates a new Container instance from the provided values // or an error if envRules validation fails. -func NewContainer(command, layers []string, envRules []EnvRuleConfig, workingDir string) (*Container, error) { +func NewContainer(command, layers []string, envRules []EnvRuleConfig, workingDir string, eMounts []string) (*Container, error) { if err := validateEnvRules(envRules); err != nil { return nil, err } return &Container{ - Command: newCommandArgs(command), - Layers: newLayers(layers), - EnvRules: newEnvRules(envRules), - WorkingDir: workingDir, + Command: newCommandArgs(command), + Layers: newLayers(layers), + EnvRules: newEnvRules(envRules), + WorkingDir: workingDir, + ExpectedMounts: newExpectedMounts(eMounts), }, nil } @@ -268,6 +281,16 @@ func newLayers(ls []string) Layers { } } +func newExpectedMounts(em []string) ExpectedMounts { + mounts := map[string]string{} + for i, m := range em { + mounts[strconv.Itoa(i)] = m + } + return ExpectedMounts{ + Elements: mounts, + } +} + // Custom JSON marshalling to add `lenth` field that matches the number of // elements present in the `elements` field. func (c Containers) MarshalJSON() ([]byte, error) { @@ -313,3 +336,14 @@ func (e EnvRules) MarshalJSON() ([]byte, error) { Alias: (*Alias)(&e), }) } + +func (em ExpectedMounts) MarshalJSON() ([]byte, error) { + type Alias ExpectedMounts + return json.Marshal(&struct { + Length int `json:"length"` + *Alias + }{ + Length: len(em.Elements), + Alias: (*Alias)(&em), + }) +} diff --git a/pkg/securitypolicy/securitypolicyenforcer.go b/pkg/securitypolicy/securitypolicyenforcer.go index d5daccbb98..b435080e9e 100644 --- a/pkg/securitypolicy/securitypolicyenforcer.go +++ b/pkg/securitypolicy/securitypolicyenforcer.go @@ -3,11 +3,17 @@ package securitypolicy import ( "errors" "fmt" + "os" + "path/filepath" "regexp" "strconv" + "strings" "sync" + "github.com/Microsoft/hcsshim/internal/hooks" + "github.com/Microsoft/hcsshim/pkg/annotations" "github.com/google/go-cmp/cmp" + oci "github.com/opencontainers/runtime-spec/specs-go" ) type SecurityPolicyEnforcer interface { @@ -15,6 +21,7 @@ type SecurityPolicyEnforcer interface { EnforceDeviceUnmountPolicy(unmountTarget string) (err error) EnforceOverlayMountPolicy(containerID string, layerPaths []string) (err error) EnforceCreateContainerPolicy(containerID string, argList []string, envList []string, workingDir string) (err error) + EnforceExpectedMountsPolicy(containerID string, spec *oci.Spec) error } func NewSecurityPolicyEnforcer(state SecurityPolicyState) (SecurityPolicyEnforcer, error) { @@ -160,13 +167,19 @@ func (c Container) toInternal() (securityPolicyContainer, error) { return securityPolicyContainer{}, err } + expectedMounts, err := c.ExpectedMounts.toInternal() + if err != nil { + return securityPolicyContainer{}, err + } + return securityPolicyContainer{ Command: command, EnvRules: envRules, Layers: layers, // No need to have toInternal(), because WorkingDir is a string both // internally and in the policy. - WorkingDir: c.WorkingDir, + WorkingDir: c.WorkingDir, + ExpectedMounts: expectedMounts, }, nil } @@ -205,6 +218,14 @@ func (l Layers) toInternal() ([]string, error) { return stringMapToStringArray(l.Elements), nil } +func (em *ExpectedMounts) toInternal() ([]string, error) { + if em.Length != len(em.Elements) { + return nil, fmt.Errorf("expectedMounts numbers don't match in policy. expected: %d, actual: %d", em.Length, len(em.Elements)) + } + + return stringMapToStringArray(em.Elements), nil +} + func stringMapToStringArray(in map[string]string) []string { inLength := len(in) out := make([]string, inLength) @@ -463,7 +484,7 @@ func equalForOverlay(a1 []string, a2 []string) bool { } func possibleIndicesForID(containerID string, mapping map[int]map[string]struct{}) []int { - possibles := []int{} + var possibles []int for index, ids := range mapping { for id := range ids { if containerID == id { @@ -475,6 +496,92 @@ func possibleIndicesForID(containerID string, mapping map[int]map[string]struct{ return possibles } +// EnforceExpectedMountsPolicy for StandardSecurityPolicyEnforcer injects a +// hooks.CreateRuntime hook into container spec and the hook ensures that +// the expected mounts appear prior container start. At the moment enforcement +// is expected to take place inside LCOW UVM. +// +// Expected mount is provided as a path under a sandbox mount path inside +// container, e.g., sandbox mount is at path "/path/in/container" and wait path +// is "/path/in/container/wait/path", which corresponds to +// "/run/gcs/c//sandboxMounts/path/on/the/host/wait/path" +// +// Iterates through container mounts to identify the correct sandbox +// mount where the wait path is nested under. The mount spec will +// be something like: +// { +// "source": "/run/gcs/c//sandboxMounts/path/on/host", +// "destination": "/path/in/container" +// } +// The wait path will be "/path/in/container/wait/path". To find the corresponding +// sandbox mount do a prefix match on wait path against all container mounts +// Destination and resolve the full path inside UVM. For example above it becomes +// "/run/gcs/c//sandboxMounts/path/on/host/wait/path" +func (pe *StandardSecurityPolicyEnforcer) EnforceExpectedMountsPolicy(containerID string, spec *oci.Spec) error { + pe.mutex.Lock() + defer pe.mutex.Unlock() + + if len(pe.Containers) < 1 { + return errors.New("policy doesn't allow mounting containers") + } + + sandboxID := spec.Annotations[annotations.KubernetesSandboxID] + if sandboxID == "" { + return errors.New("no sandbox ID present in spec annotations") + } + + var wMounts []string + pIndices := possibleIndicesForID(containerID, pe.ContainerIndexToContainerIds) + if len(pIndices) == 0 { + return errors.New("no valid container indices found") + } + + // Unlike environment variable and command line enforcement, there isn't anything + // to validate here, since we're essentially just injecting hooks when necessary + // for all containers. + matchFound := false + for _, index := range pIndices { + if !matchFound { + matchFound = true + wMounts = pe.Containers[index].ExpectedMounts + } else { + pe.narrowMatchesForContainerIndex(index, containerID) + } + } + + if len(wMounts) == 0 { + return nil + } + + var wPaths []string + for _, mount := range wMounts { + var wp string + for _, m := range spec.Mounts { + // prefix matching to find correct sandbox mount + if strings.HasPrefix(mount, m.Destination) { + wp = filepath.Join(m.Source, strings.TrimPrefix(mount, m.Destination)) + break + } + } + if wp == "" { + return fmt.Errorf("invalid mount path: %q", mount) + } + wPaths = append(wPaths, filepath.Clean(wp)) + } + + pathsArg := strings.Join(wPaths, ",") + waitPathsBinary := "/bin/wait-paths" + args := []string{ + waitPathsBinary, + "--paths", + pathsArg, + "--timeout", + "60", + } + hook := hooks.NewOCIHook(waitPathsBinary, args, os.Environ()) + return hooks.AddOCIHook(spec, hooks.CreateRuntime, hook) +} + type OpenDoorSecurityPolicyEnforcer struct{} var _ SecurityPolicyEnforcer = (*OpenDoorSecurityPolicyEnforcer)(nil) @@ -495,6 +602,10 @@ func (p *OpenDoorSecurityPolicyEnforcer) EnforceCreateContainerPolicy(_ string, return nil } +func (p *OpenDoorSecurityPolicyEnforcer) EnforceExpectedMountsPolicy(_ string, _ *oci.Spec) error { + return nil +} + type ClosedDoorSecurityPolicyEnforcer struct{} var _ SecurityPolicyEnforcer = (*ClosedDoorSecurityPolicyEnforcer)(nil) @@ -514,3 +625,7 @@ func (p *ClosedDoorSecurityPolicyEnforcer) EnforceOverlayMountPolicy(containerID func (p *ClosedDoorSecurityPolicyEnforcer) EnforceCreateContainerPolicy(_ string, _ []string, _ []string, _ string) (err error) { return errors.New("running commands is denied by policy") } + +func (p *ClosedDoorSecurityPolicyEnforcer) EnforceExpectedMountsPolicy(_ string, _ *oci.Spec) error { + return errors.New("enforcing expected mounts is denied by policy") +} diff --git a/test/cri-containerd/container_test.go b/test/cri-containerd/container_test.go index 52ae6b0efd..1afbc1b7ce 100644 --- a/test/cri-containerd/container_test.go +++ b/test/cri-containerd/container_test.go @@ -6,6 +6,7 @@ package cri_containerd import ( "bufio" "context" + "fmt" "io" "os" "path/filepath" @@ -14,9 +15,11 @@ import ( "testing" "time" - "github.com/Microsoft/hcsshim/pkg/annotations" "github.com/sirupsen/logrus" runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" + + "github.com/Microsoft/hcsshim/internal/guestpath" + "github.com/Microsoft/hcsshim/pkg/annotations" ) func runLogRotationContainer(t *testing.T, sandboxRequest *runtime.RunPodSandboxRequest, request *runtime.CreateContainerRequest, log string, logArchive string) { @@ -725,7 +728,7 @@ func Test_CreateContainer_HugePageMount_LCOW(t *testing.T) { }, Mounts: []*runtime.Mount{ { - HostPath: "hugepages://2M/hugepage2M", + HostPath: fmt.Sprintf("%s2M/hugepage2M", guestpath.HugePagesMountPrefix), ContainerPath: "/mnt/hugepage2M", Readonly: false, Propagation: runtime.MountPropagation_PROPAGATION_BIDIRECTIONAL, diff --git a/test/cri-containerd/policy_test.go b/test/cri-containerd/policy_test.go index b4dfa87777..f46f34b555 100644 --- a/test/cri-containerd/policy_test.go +++ b/test/cri-containerd/policy_test.go @@ -5,6 +5,8 @@ package cri_containerd import ( "context" + "fmt" + "strings" "testing" "github.com/Microsoft/hcsshim/internal/tools/securitypolicy/helpers" @@ -17,6 +19,15 @@ var ( validPolicyAlpineCommand = []string{"ash", "-c", "echo 'Hello'"} ) +type configOpt func(*securitypolicy.ContainerConfig) error + +func withExpectedMounts(em []string) configOpt { + return func(conf *securitypolicy.ContainerConfig) error { + conf.ExpectedMounts = append(conf.ExpectedMounts, em...) + return nil + } +} + func securityPolicyFromContainers(containers []securitypolicy.ContainerConfig) (string, error) { pc, err := helpers.PolicyContainersFromConfigs(containers) if err != nil { @@ -47,6 +58,7 @@ func alpineSecurityPolicy(t *testing.T) string { []securitypolicy.EnvRuleConfig{}, securitypolicy.AuthConfig{}, "", + []string{}, ) containers := append(defaultContainers, alpineContainer) @@ -116,3 +128,135 @@ func Test_RunSimpleAlpineContainer_WithPolicy_Allowed(t *testing.T) { startContainer(t, client, ctx, containerID) stopContainer(t, client, ctx, containerID) } + +func syncContainerConfigs(writePath, waitPath string) (writer, waiter *securitypolicy.ContainerConfig) { + writerCmdArgs := []string{"ash", "-c", fmt.Sprintf("touch %s && while true; do echo hello1; sleep 1; done", writePath)} + writer = &securitypolicy.ContainerConfig{ + ImageName: "alpine:latest", + Command: writerCmdArgs, + } + // create container #2 that waits for a path to appear + echoCmdArgs := []string{"ash", "-c", "while true; do echo hello2; sleep 1; done"} + waiter = &securitypolicy.ContainerConfig{ + ImageName: "alpine:latest", + Command: echoCmdArgs, + ExpectedMounts: []string{waitPath}, + } + return writer, waiter +} + +func syncContainerRequests( + writer, waiter *securitypolicy.ContainerConfig, + podID string, + podConfig *v1alpha2.PodSandboxConfig, +) (writerReq, waiterReq *v1alpha2.CreateContainerRequest) { + writerReq = getCreateContainerRequest( + podID, + "alpine-writer", + "alpine:latest", + writer.Command, + podConfig, + ) + writerReq.Config.Mounts = append(writerReq.Config.Mounts, &v1alpha2.Mount{ + HostPath: "sandbox://host/path", + ContainerPath: "/mnt/shared/container-A", + }) + + waiterReq = getCreateContainerRequest( + podID, + "alpine-waiter", + "alpine:latest", + waiter.Command, + podConfig, + ) + waiterReq.Config.Mounts = append(waiterReq.Config.Mounts, &v1alpha2.Mount{ + // The HostPath must be the same as for the "writer" container + HostPath: "sandbox://host/path", + ContainerPath: "/mnt/shared/container-B", + }) + + return writerReq, waiterReq +} + +func Test_RunContainers_WithSyncHooks_ValidWaitPath(t *testing.T) { + requireFeatures(t, featureLCOW, featureLCOWIntegrity) + + writerCfg, waiterCfg := syncContainerConfigs( + "/mnt/shared/container-A/sync-file", "/mnt/shared/container-B/sync-file") + + containerConfigs := append(helpers.DefaultContainerConfigs(), *writerCfg, *waiterCfg) + policyString, err := securityPolicyFromContainers(containerConfigs) + if err != nil { + t.Fatalf("failed to generate security policy string: %s", err) + } + + client := newTestRuntimeClient(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // create pod with security policy + podRequest := sandboxRequestWithPolicy(t, policyString) + podID := runPodSandbox(t, client, ctx, podRequest) + defer removePodSandbox(t, client, ctx, podID) + defer stopPodSandbox(t, client, ctx, podID) + + writerReq, waiterReq := syncContainerRequests(writerCfg, waiterCfg, podID, podRequest.Config) + + cidWriter := createContainer(t, client, ctx, writerReq) + cidWaiter := createContainer(t, client, ctx, waiterReq) + + startContainer(t, client, ctx, cidWriter) + defer removeContainer(t, client, ctx, cidWriter) + defer stopContainer(t, client, ctx, cidWriter) + + startContainer(t, client, ctx, cidWaiter) + defer removeContainer(t, client, ctx, cidWaiter) + defer stopContainer(t, client, ctx, cidWaiter) +} + +func Test_RunContainers_WithSyncHooks_InvalidWaitPath(t *testing.T) { + requireFeatures(t, featureLCOW, featureLCOWIntegrity) + + writerCfg, waiterCfg := syncContainerConfigs( + "/mnt/shared/container-A/sync-file", + "/mnt/shared/container-B/sync-file-invalid", // NOTE: this is an invalid wait path + ) + + containerConfigs := append(helpers.DefaultContainerConfigs(), *writerCfg, *waiterCfg) + policyString, err := securityPolicyFromContainers(containerConfigs) + if err != nil { + t.Fatalf("failed to generate security policy string: %s", policyString) + } + + client := newTestRuntimeClient(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // create pod with security policy + podRequest := sandboxRequestWithPolicy(t, policyString) + podID := runPodSandbox(t, client, ctx, podRequest) + defer removePodSandbox(t, client, ctx, podID) + defer stopPodSandbox(t, client, ctx, podID) + + writerReq, waiterReq := syncContainerRequests(writerCfg, waiterCfg, podID, podRequest.Config) + cidWriter := createContainer(t, client, ctx, writerReq) + cidWaiter := createContainer(t, client, ctx, waiterReq) + + startContainer(t, client, ctx, cidWriter) + defer removeContainer(t, client, ctx, cidWriter) + defer stopContainer(t, client, ctx, cidWriter) + + _, err = client.StartContainer(ctx, &v1alpha2.StartContainerRequest{ + ContainerId: cidWaiter, + }) + expectedErrString := "timeout while waiting for path" + if err == nil { + defer removeContainer(t, client, ctx, cidWaiter) + defer stopContainer(t, client, ctx, cidWaiter) + t.Fatalf("should fail, succeeded instead") + } else { + if !strings.Contains(err.Error(), expectedErrString) { + t.Fatalf("expected error: %q, got: %q", expectedErrString, err) + } + } +} diff --git a/test/vendor/github.com/Microsoft/hcsshim/Makefile b/test/vendor/github.com/Microsoft/hcsshim/Makefile index ac00def635..c09b1e192a 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/Makefile +++ b/test/vendor/github.com/Microsoft/hcsshim/Makefile @@ -32,7 +32,7 @@ clean: test: cd $(SRCROOT) && go test -v ./internal/guest/... -out/delta.tar.gz: bin/init bin/vsockexec bin/cmd/gcs bin/cmd/gcstools Makefile +out/delta.tar.gz: bin/init bin/vsockexec bin/cmd/gcs bin/cmd/gcstools bin/cmd/hooks/wait-paths Makefile @mkdir -p out rm -rf rootfs mkdir -p rootfs/bin/ @@ -40,6 +40,7 @@ out/delta.tar.gz: bin/init bin/vsockexec bin/cmd/gcs bin/cmd/gcstools Makefile cp bin/vsockexec rootfs/bin/ cp bin/cmd/gcs rootfs/bin/ cp bin/cmd/gcstools rootfs/bin/ + cp bin/cmd/hooks/wait-paths rootfs/bin/ for tool in $(GCS_TOOLS); do ln -s gcstools rootfs/bin/$$tool; done git -C $(SRCROOT) rev-parse HEAD > rootfs/gcs.commit && \ git -C $(SRCROOT) rev-parse --abbrev-ref HEAD > rootfs/gcs.branch @@ -60,6 +61,7 @@ out/initrd.img: $(BASE) out/delta.tar.gz $(SRCROOT)/hack/catcpio.sh -include deps/cmd/gcs.gomake -include deps/cmd/gcstools.gomake +-include deps/cmd/hooks/wait-paths.gomake # Implicit rule for includes that define Go targets. %.gomake: $(SRCROOT)/Makefile diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/devices/drivers.go b/test/vendor/github.com/Microsoft/hcsshim/internal/devices/drivers.go index 4efe677e22..0dae33c963 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/devices/drivers.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/devices/drivers.go @@ -8,6 +8,7 @@ import ( "fmt" "github.com/Microsoft/hcsshim/internal/cmd" + "github.com/Microsoft/hcsshim/internal/guestpath" "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/resources" "github.com/Microsoft/hcsshim/internal/uvm" @@ -45,7 +46,7 @@ func InstallKernelDriver(ctx context.Context, vm *uvm.UtilityVM, driver string) } return closer, execPnPInstallDriver(ctx, vm, uvmPath) } - uvmPathForShare := fmt.Sprintf(uvm.LCOWGlobalMountPrefix, vm.UVMMountCounter()) + uvmPathForShare := fmt.Sprintf(guestpath.LCOWGlobalMountPrefixFmt, vm.UVMMountCounter()) scsiCloser, err := vm.AddSCSI(ctx, driver, uvmPathForShare, true, false, []string{}, uvm.VMAccessTypeIndividual) if err != nil { return closer, fmt.Errorf("failed to add SCSI disk to utility VM for path %+v: %s", driver, err) diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/guestpath/paths.go b/test/vendor/github.com/Microsoft/hcsshim/internal/guestpath/paths.go new file mode 100644 index 0000000000..62bbfeb636 --- /dev/null +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/guestpath/paths.go @@ -0,0 +1,24 @@ +package guestpath + +const ( + // LCOWNvidiaMountPath is the path format in LCOW UVM where nvidia tools are mounted + // keep this value in sync with opengcs + LCOWNvidiaMountPath = "/run/nvidia" + // LCOWRootPrefixInUVM is the path inside UVM where LCOW container's root file system will be mounted + LCOWRootPrefixInUVM = "/run/gcs/c" + // WCOWRootPrefixInUVM is the path inside UVM where WCOW container's root file system will be mounted + WCOWRootPrefixInUVM = `C:\c` + // SandboxMountPrefix is mount prefix used in container spec to mark a sandbox-mount + SandboxMountPrefix = "sandbox://" + // HugePagesMountPrefix is mount prefix used in container spec to mark a huge-pages mount + HugePagesMountPrefix = "hugepages://" + // LCOWMountPathPrefixFmt is the path format in the LCOW UVM where non global mounts, such + // as Plan9 mounts are added + LCOWMountPathPrefixFmt = "/mounts/m%d" + // LCOWGlobalMountPrefixFmt is the path format in the LCOW UVM where global mounts are added + LCOWGlobalMountPrefixFmt = "/run/mounts/m%d" + // WCOWGlobalMountPrefixFmt is the path prefix format in the WCOW UVM where mounts are added + WCOWGlobalMountPrefixFmt = "C:\\mounts\\m%d" + // RootfsPath is part of the container's rootfs path + RootfsPath = "rootfs" +) diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/create.go b/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/create.go index 16821a1861..058530aac1 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/create.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/create.go @@ -14,6 +14,7 @@ import ( "github.com/Microsoft/go-winio/pkg/guid" "github.com/Microsoft/hcsshim/internal/clone" "github.com/Microsoft/hcsshim/internal/cow" + "github.com/Microsoft/hcsshim/internal/guestpath" "github.com/Microsoft/hcsshim/internal/hcs" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" "github.com/Microsoft/hcsshim/internal/log" @@ -27,8 +28,8 @@ import ( ) var ( - lcowRootInUVM = "/run/gcs/c/%s" - wcowRootInUVM = `C:\c\%s` + lcowRootInUVM = guestpath.LCOWRootPrefixInUVM + "/%s" + wcowRootInUVM = guestpath.WCOWRootPrefixInUVM + "/%s" ) // CreateOptions are the set of fields used to call CreateContainer(). diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/devices.go b/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/devices.go index 42d11aac52..4f17a715d6 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/devices.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/devices.go @@ -9,7 +9,11 @@ import ( "path/filepath" "strconv" + specs "github.com/opencontainers/runtime-spec/specs-go" + "github.com/pkg/errors" + "github.com/Microsoft/hcsshim/internal/devices" + "github.com/Microsoft/hcsshim/internal/guestpath" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/oci" @@ -17,8 +21,6 @@ import ( "github.com/Microsoft/hcsshim/internal/uvm" "github.com/Microsoft/hcsshim/osversion" "github.com/Microsoft/hcsshim/pkg/annotations" - specs "github.com/opencontainers/runtime-spec/specs-go" - "github.com/pkg/errors" ) const deviceUtilExeName = "device-util.exe" @@ -222,14 +224,14 @@ func handleAssignedDevicesLCOW( scsiMount, err := vm.AddSCSI( ctx, gpuSupportVhdPath, - uvm.LCOWNvidiaMountPath, + guestpath.LCOWNvidiaMountPath, true, false, options, uvm.VMAccessTypeNoop, ) if err != nil { - return resultDevs, closers, errors.Wrapf(err, "failed to add scsi device %s in the UVM %s at %s", gpuSupportVhdPath, vm.ID(), uvm.LCOWNvidiaMountPath) + return resultDevs, closers, errors.Wrapf(err, "failed to add scsi device %s in the UVM %s at %s", gpuSupportVhdPath, vm.ID(), guestpath.LCOWNvidiaMountPath) } closers = append(closers, scsiMount) } diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/hcsdoc_wcow.go b/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/hcsdoc_wcow.go index 688d901609..b3080399a6 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/hcsdoc_wcow.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/hcsdoc_wcow.go @@ -11,6 +11,10 @@ import ( "regexp" "strings" + specs "github.com/opencontainers/runtime-spec/specs-go" + "github.com/sirupsen/logrus" + + "github.com/Microsoft/hcsshim/internal/guestpath" "github.com/Microsoft/hcsshim/internal/hcs/schema1" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" "github.com/Microsoft/hcsshim/internal/layers" @@ -22,8 +26,6 @@ import ( "github.com/Microsoft/hcsshim/internal/wclayer" "github.com/Microsoft/hcsshim/osversion" "github.com/Microsoft/hcsshim/pkg/annotations" - specs "github.com/opencontainers/runtime-spec/specs-go" - "github.com/sirupsen/logrus" ) // A simple wrapper struct around the container mount configs that should be added to the @@ -78,7 +80,7 @@ func createMountsConfig(ctx context.Context, coi *createOptionsInternal) (*mount return nil, err } mdv2.HostPath = uvmPath - } else if strings.HasPrefix(mount.Source, "sandbox://") { + } else if strings.HasPrefix(mount.Source, guestpath.SandboxMountPrefix) { // Convert to the path in the guest that was asked for. mdv2.HostPath = convertToWCOWSandboxMountPath(mount.Source) } else { diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/resources_lcow.go b/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/resources_lcow.go index 085854a4c7..d438738c28 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/resources_lcow.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/resources_lcow.go @@ -13,12 +13,14 @@ import ( "path/filepath" "strings" + specs "github.com/opencontainers/runtime-spec/specs-go" + "github.com/pkg/errors" + + "github.com/Microsoft/hcsshim/internal/guestpath" "github.com/Microsoft/hcsshim/internal/layers" "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/resources" "github.com/Microsoft/hcsshim/internal/uvm" - specs "github.com/opencontainers/runtime-spec/specs-go" - "github.com/pkg/errors" ) func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r *resources.Resources, isSandbox bool) error { @@ -39,7 +41,7 @@ func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r * // This is the "Plan 9" root filesystem. // TODO: We need a test for this. Ask @jstarks how you can even lay this out on Windows. hostPath := coi.Spec.Root.Path - uvmPathForContainersFileSystem := path.Join(r.ContainerRootInUVM(), uvm.RootfsPath) + uvmPathForContainersFileSystem := path.Join(r.ContainerRootInUVM(), guestpath.RootfsPath) share, err := coi.HostingSystem.AddPlan9(ctx, hostPath, uvmPathForContainersFileSystem, coi.Spec.Root.Readonly, false, nil) if err != nil { return errors.Wrap(err, "adding plan9 root") @@ -65,7 +67,7 @@ func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r * if coi.HostingSystem != nil { hostPath := mount.Source - uvmPathForShare := path.Join(containerRootInUVM, fmt.Sprintf(uvm.LCOWMountPathPrefix, i)) + uvmPathForShare := path.Join(containerRootInUVM, fmt.Sprintf(guestpath.LCOWMountPathPrefixFmt, i)) uvmPathForFile := uvmPathForShare readOnly := false @@ -79,7 +81,7 @@ func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r * l := log.G(ctx).WithField("mount", fmt.Sprintf("%+v", mount)) if mount.Type == "physical-disk" { l.Debug("hcsshim::allocateLinuxResources Hot-adding SCSI physical disk for OCI mount") - uvmPathForShare = fmt.Sprintf(uvm.LCOWGlobalMountPrefix, coi.HostingSystem.UVMMountCounter()) + uvmPathForShare = fmt.Sprintf(guestpath.LCOWGlobalMountPrefixFmt, coi.HostingSystem.UVMMountCounter()) scsiMount, err := coi.HostingSystem.AddSCSIPhysicalDisk(ctx, hostPath, uvmPathForShare, readOnly, mount.Options) if err != nil { return errors.Wrapf(err, "adding SCSI physical disk mount %+v", mount) @@ -90,7 +92,7 @@ func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r * coi.Spec.Mounts[i].Type = "none" } else if mount.Type == "virtual-disk" { l.Debug("hcsshim::allocateLinuxResources Hot-adding SCSI virtual disk for OCI mount") - uvmPathForShare = fmt.Sprintf(uvm.LCOWGlobalMountPrefix, coi.HostingSystem.UVMMountCounter()) + uvmPathForShare = fmt.Sprintf(guestpath.LCOWGlobalMountPrefixFmt, coi.HostingSystem.UVMMountCounter()) // if the scsi device is already attached then we take the uvm path that the function below returns // that is where it was previously mounted in UVM @@ -110,15 +112,19 @@ func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r * uvmPathForFile = scsiMount.UVMPath r.Add(scsiMount) coi.Spec.Mounts[i].Type = "none" - } else if strings.HasPrefix(mount.Source, "sandbox://") { + } else if strings.HasPrefix(mount.Source, guestpath.SandboxMountPrefix) { // Mounts that map to a path in UVM are specified with 'sandbox://' prefix. // example: sandbox:///a/dirInUvm destination:/b/dirInContainer uvmPathForFile = mount.Source - } else if strings.HasPrefix(mount.Source, "hugepages://") { + } else if strings.HasPrefix(mount.Source, guestpath.HugePagesMountPrefix) { // currently we only support 2M hugepage size - hugePageSubDirs := strings.Split(strings.TrimPrefix(mount.Source, "hugepages://"), "/") + hugePageSubDirs := strings.Split(strings.TrimPrefix(mount.Source, guestpath.HugePagesMountPrefix), "/") if len(hugePageSubDirs) < 2 { - return errors.Errorf(`%s mount path is invalid, expected format: hugepages:///`, mount.Source) + return errors.Errorf( + `%s mount path is invalid, expected format: %s/`, + mount.Source, + guestpath.HugePagesMountPrefix, + ) } // hugepages:// should be followed by pagesize @@ -155,15 +161,17 @@ func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r * } } - if coi.HostingSystem != nil { - if coi.hasWindowsAssignedDevices() { - windowsDevices, closers, err := handleAssignedDevicesLCOW(ctx, coi.HostingSystem, coi.Spec.Annotations, coi.Spec.Windows.Devices) - if err != nil { - return err - } - r.Add(closers...) - coi.Spec.Windows.Devices = windowsDevices + if coi.HostingSystem == nil { + return nil + } + + if coi.hasWindowsAssignedDevices() { + windowsDevices, closers, err := handleAssignedDevicesLCOW(ctx, coi.HostingSystem, coi.Spec.Annotations, coi.Spec.Windows.Devices) + if err != nil { + return err } + r.Add(closers...) + coi.Spec.Windows.Devices = windowsDevices } return nil } diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/resources_wcow.go b/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/resources_wcow.go index 1150ca5c3c..fa22c8047e 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/resources_wcow.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/resources_wcow.go @@ -13,16 +13,18 @@ import ( "path/filepath" "strings" + specs "github.com/opencontainers/runtime-spec/specs-go" + "github.com/pkg/errors" + "github.com/Microsoft/hcsshim/internal/cmd" "github.com/Microsoft/hcsshim/internal/credentials" + "github.com/Microsoft/hcsshim/internal/guestpath" "github.com/Microsoft/hcsshim/internal/layers" "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/resources" "github.com/Microsoft/hcsshim/internal/schemaversion" "github.com/Microsoft/hcsshim/internal/uvm" "github.com/Microsoft/hcsshim/internal/wclayer" - specs "github.com/opencontainers/runtime-spec/specs-go" - "github.com/pkg/errors" ) const wcowSandboxMountPath = "C:\\SandboxMounts" @@ -140,7 +142,7 @@ func setupMounts(ctx context.Context, coi *createOptionsInternal, r *resources.R } if coi.HostingSystem != nil && schemaversion.IsV21(coi.actualSchemaVersion) { - uvmPath := fmt.Sprintf(uvm.WCOWGlobalMountPrefix, coi.HostingSystem.UVMMountCounter()) + uvmPath := fmt.Sprintf(guestpath.WCOWGlobalMountPrefixFmt, coi.HostingSystem.UVMMountCounter()) readOnly := false for _, o := range mount.Options { if strings.ToLower(o) == "ro" { @@ -178,7 +180,7 @@ func setupMounts(ctx context.Context, coi *createOptionsInternal, r *resources.R return errors.Wrapf(err, "adding SCSI EVD mount failed %+v", mount) } r.Add(scsiMount) - } else if strings.HasPrefix(mount.Source, "sandbox://") { + } else if strings.HasPrefix(mount.Source, guestpath.SandboxMountPrefix) { // Mounts that map to a path in the UVM are specified with a 'sandbox://' prefix. // // Example: sandbox:///a/dirInUvm destination:C:\\dirInContainer. @@ -228,6 +230,6 @@ func setupMounts(ctx context.Context, coi *createOptionsInternal, r *resources.R } func convertToWCOWSandboxMountPath(source string) string { - subPath := strings.TrimPrefix(source, "sandbox://") + subPath := strings.TrimPrefix(source, guestpath.SandboxMountPrefix) return filepath.Join(wcowSandboxMountPath, subPath) } diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/hooks/spec.go b/test/vendor/github.com/Microsoft/hcsshim/internal/hooks/spec.go new file mode 100644 index 0000000000..51ba3aa592 --- /dev/null +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/hooks/spec.go @@ -0,0 +1,53 @@ +package hooks + +import ( + "fmt" + + oci "github.com/opencontainers/runtime-spec/specs-go" +) + +// Note: The below type definition as well as constants have been copied from +// https://github.com/opencontainers/runc/blob/master/libcontainer/configs/config.go. +// This is done to not introduce a direct dependency on runc, which would complicate +// integration with windows. +type HookName string + +const ( + + // Prestart commands are executed after the container namespaces are created, + // but before the user supplied command is executed from init. + // Note: This hook is now deprecated + // Prestart commands are called in the Runtime namespace. + Prestart HookName = "prestart" + + // CreateRuntime commands MUST be called as part of the create operation after + // the runtime environment has been created but before the pivot_root has been executed. + // CreateRuntime is called immediately after the deprecated Prestart hook. + // CreateRuntime commands are called in the Runtime Namespace. + CreateRuntime HookName = "createRuntime" +) + +// NewOCIHook creates a new oci.Hook with given parameters +func NewOCIHook(path string, args, env []string) oci.Hook { + return oci.Hook{ + Path: path, + Args: args, + Env: env, + } +} + +// AddOCIHook adds oci.Hook of the given hook name to spec +func AddOCIHook(spec *oci.Spec, hn HookName, hk oci.Hook) error { + if spec.Hooks == nil { + spec.Hooks = &oci.Hooks{} + } + switch hn { + case Prestart: + spec.Hooks.Prestart = append(spec.Hooks.Prestart, hk) + case CreateRuntime: + spec.Hooks.CreateRuntime = append(spec.Hooks.CreateRuntime, hk) + default: + return fmt.Errorf("hook %q is not supported", hn) + } + return nil +} diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/layers/layers.go b/test/vendor/github.com/Microsoft/hcsshim/internal/layers/layers.go index 573ad72aa7..0334076365 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/layers/layers.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/layers/layers.go @@ -11,15 +11,17 @@ import ( "path/filepath" "time" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "golang.org/x/sys/windows" + + "github.com/Microsoft/hcsshim/internal/guestpath" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" "github.com/Microsoft/hcsshim/internal/hcserror" "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/ospath" "github.com/Microsoft/hcsshim/internal/uvm" "github.com/Microsoft/hcsshim/internal/wclayer" - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "golang.org/x/sys/windows" ) // ImageLayers contains all the layers for an image. @@ -261,7 +263,7 @@ func MountContainerLayers(ctx context.Context, containerID string, layerFolders err = vm.CombineLayersWCOW(ctx, layers, containerScratchPathInUVM) rootfs = containerScratchPathInUVM } else { - rootfs = ospath.Join(vm.OS(), guestRoot, uvm.RootfsPath) + rootfs = ospath.Join(vm.OS(), guestRoot, guestpath.RootfsPath) err = vm.CombineLayersLCOW(ctx, containerID, lcowUvmLayerPaths, containerScratchPathInUVM, rootfs) } if err != nil { @@ -289,7 +291,7 @@ func addLCOWLayer(ctx context.Context, vm *uvm.UtilityVM, layerPath string) (uvm } options := []string{"ro"} - uvmPath = fmt.Sprintf(uvm.LCOWGlobalMountPrefix, vm.UVMMountCounter()) + uvmPath = fmt.Sprintf(guestpath.LCOWGlobalMountPrefixFmt, vm.UVMMountCounter()) sm, err := vm.AddSCSI(ctx, layerPath, uvmPath, true, false, options, uvm.VMAccessTypeNoop) if err != nil { return "", fmt.Errorf("failed to add SCSI layer: %s", err) @@ -460,7 +462,7 @@ func containerRootfsPath(vm *uvm.UtilityVM, rootPath string) string { if vm.OS() == "windows" { return ospath.Join(vm.OS(), rootPath) } - return ospath.Join(vm.OS(), rootPath, uvm.RootfsPath) + return ospath.Join(vm.OS(), rootPath, guestpath.RootfsPath) } func getScratchVHDPath(layerFolders []string) (string, error) { diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/tools/securitypolicy/helpers/helpers.go b/test/vendor/github.com/Microsoft/hcsshim/internal/tools/securitypolicy/helpers/helpers.go index 614e5c10e8..103b2bae3a 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/tools/securitypolicy/helpers/helpers.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/tools/securitypolicy/helpers/helpers.go @@ -74,6 +74,7 @@ func DefaultContainerConfigs() []securitypolicy.ContainerConfig { []securitypolicy.EnvRuleConfig{}, securitypolicy.AuthConfig{}, "", + []string{}, ) return []securitypolicy.ContainerConfig{pause} } @@ -136,7 +137,13 @@ func PolicyContainersFromConfigs(containerConfigs []securitypolicy.ContainerConf workingDir = containerConfig.WorkingDir } - container, err := securitypolicy.NewContainer(containerConfig.Command, layerHashes, envRules, workingDir) + container, err := securitypolicy.NewContainer( + containerConfig.Command, + layerHashes, + envRules, + workingDir, + containerConfig.ExpectedMounts, + ) if err != nil { return nil, err } diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/uvm/constants.go b/test/vendor/github.com/Microsoft/hcsshim/internal/uvm/constants.go index 842a8a5e8c..1ddcf903ab 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/uvm/constants.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/uvm/constants.go @@ -16,19 +16,6 @@ const ( // DefaultVPMemSizeBytes is the default size of a VPMem device if the create request // doesn't specify. DefaultVPMemSizeBytes = 4 * 1024 * 1024 * 1024 // 4GB - - // LCOWMountPathPrefix is the path format in the LCOW UVM where non global mounts, such - // as Plan9 mounts are added - LCOWMountPathPrefix = "/mounts/m%d" - // LCOWGlobalMountPrefix is the path format in the LCOW UVM where global mounts are added - LCOWGlobalMountPrefix = "/run/mounts/m%d" - // LCOWNvidiaMountPath is the path format in LCOW UVM where nvidia tools are mounted - // keep this value in sync with opengcs - LCOWNvidiaMountPath = "/run/nvidia" - // WCOWGlobalMountPrefix is the path prefix format in the WCOW UVM where mounts are added - WCOWGlobalMountPrefix = "C:\\mounts\\m%d" - // RootfsPath is part of the container's rootfs path - RootfsPath = "rootfs" ) var ( diff --git a/test/vendor/github.com/Microsoft/hcsshim/pkg/annotations/annotations.go b/test/vendor/github.com/Microsoft/hcsshim/pkg/annotations/annotations.go index 7a51b2a0e4..2f06dfb8bd 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/pkg/annotations/annotations.go +++ b/test/vendor/github.com/Microsoft/hcsshim/pkg/annotations/annotations.go @@ -261,7 +261,7 @@ const ( // GuestStateFile specifies the path of the vmgs file to use if required. Only applies in SNP mode. GuestStateFile = "io.microsoft.virtualmachine.lcow.gueststatefile" - // AnnotationDisableLCOWTimeSyncService is used to disable the chronyd time + // DisableLCOWTimeSyncService is used to disable the chronyd time // synchronization service inside the LCOW UVM. DisableLCOWTimeSyncService = "io.microsoft.virtualmachine.lcow.timesync.disable" diff --git a/test/vendor/github.com/Microsoft/hcsshim/pkg/securitypolicy/securitypolicy.go b/test/vendor/github.com/Microsoft/hcsshim/pkg/securitypolicy/securitypolicy.go index 87cf644aef..2a539ed9db 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/pkg/securitypolicy/securitypolicy.go +++ b/test/vendor/github.com/Microsoft/hcsshim/pkg/securitypolicy/securitypolicy.go @@ -38,11 +38,12 @@ type EnvRuleConfig struct { // ContainerConfig contains toml or JSON config for container described // in security policy. type ContainerConfig struct { - ImageName string `json:"image_name" toml:"image_name"` - Command []string `json:"command" toml:"command"` - Auth AuthConfig `json:"auth" toml:"auth"` - EnvRules []EnvRuleConfig `json:"env_rules" toml:"env_rule"` - WorkingDir string `json:"working_dir" toml:"working_dir"` + ImageName string `json:"image_name" toml:"image_name"` + Command []string `json:"command" toml:"command"` + Auth AuthConfig `json:"auth" toml:"auth"` + EnvRules []EnvRuleConfig `json:"env_rules" toml:"env_rule"` + WorkingDir string `json:"working_dir" toml:"working_dir"` + ExpectedMounts []string `json:"expected_mounts" toml:"expected_mounts"` } // NewContainerConfig creates a new ContainerConfig from the given values. @@ -52,13 +53,15 @@ func NewContainerConfig( envRules []EnvRuleConfig, auth AuthConfig, workingDir string, + expectedMounts []string, ) ContainerConfig { return ContainerConfig{ - ImageName: imageName, - Command: command, - EnvRules: envRules, - Auth: auth, - WorkingDir: workingDir, + ImageName: imageName, + Command: command, + EnvRules: envRules, + Auth: auth, + WorkingDir: workingDir, + ExpectedMounts: expectedMounts, } } @@ -97,6 +100,9 @@ type securityPolicyContainer struct { // WorkingDir is a path to container's working directory, which all the processes // will default to. WorkingDir string + // Unordered list of mounts which are expected to be present when the container + // starts + ExpectedMounts []string `json:"expected_mounts"` } // SecurityPolicyState is a structure that holds user supplied policy to enforce @@ -119,7 +125,7 @@ type EncodedSecurityPolicy struct { // security policy for given policy. The security policy is transmitted as json // in an annotation, so we first have to remove the base64 encoding that allows // the JSON based policy to be passed as a string. From there, we decode the -// JSONand setup our security policy struct +// JSON and setup our security policy struct func NewSecurityPolicyState(base64Policy string) (*SecurityPolicyState, error) { // construct an encoded security policy that holds the base64 representation encodedSecurityPolicy := EncodedSecurityPolicy{ @@ -175,10 +181,11 @@ type Containers struct { } type Container struct { - Command CommandArgs `json:"command"` - EnvRules EnvRules `json:"env_rules"` - Layers Layers `json:"layers"` - WorkingDir string `json:"working_dir"` + Command CommandArgs `json:"command"` + EnvRules EnvRules `json:"env_rules"` + Layers Layers `json:"layers"` + WorkingDir string `json:"working_dir"` + ExpectedMounts ExpectedMounts `json:"expected_mounts"` } type Layers struct { @@ -198,17 +205,23 @@ type EnvRules struct { Elements map[string]EnvRuleConfig `json:"elements"` } +type ExpectedMounts struct { + Length int `json:"length"` + Elements map[string]string `json:"elements"` +} + // NewContainer creates a new Container instance from the provided values // or an error if envRules validation fails. -func NewContainer(command, layers []string, envRules []EnvRuleConfig, workingDir string) (*Container, error) { +func NewContainer(command, layers []string, envRules []EnvRuleConfig, workingDir string, eMounts []string) (*Container, error) { if err := validateEnvRules(envRules); err != nil { return nil, err } return &Container{ - Command: newCommandArgs(command), - Layers: newLayers(layers), - EnvRules: newEnvRules(envRules), - WorkingDir: workingDir, + Command: newCommandArgs(command), + Layers: newLayers(layers), + EnvRules: newEnvRules(envRules), + WorkingDir: workingDir, + ExpectedMounts: newExpectedMounts(eMounts), }, nil } @@ -268,6 +281,16 @@ func newLayers(ls []string) Layers { } } +func newExpectedMounts(em []string) ExpectedMounts { + mounts := map[string]string{} + for i, m := range em { + mounts[strconv.Itoa(i)] = m + } + return ExpectedMounts{ + Elements: mounts, + } +} + // Custom JSON marshalling to add `lenth` field that matches the number of // elements present in the `elements` field. func (c Containers) MarshalJSON() ([]byte, error) { @@ -313,3 +336,14 @@ func (e EnvRules) MarshalJSON() ([]byte, error) { Alias: (*Alias)(&e), }) } + +func (em ExpectedMounts) MarshalJSON() ([]byte, error) { + type Alias ExpectedMounts + return json.Marshal(&struct { + Length int `json:"length"` + *Alias + }{ + Length: len(em.Elements), + Alias: (*Alias)(&em), + }) +} diff --git a/test/vendor/github.com/Microsoft/hcsshim/pkg/securitypolicy/securitypolicyenforcer.go b/test/vendor/github.com/Microsoft/hcsshim/pkg/securitypolicy/securitypolicyenforcer.go index d5daccbb98..b435080e9e 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/pkg/securitypolicy/securitypolicyenforcer.go +++ b/test/vendor/github.com/Microsoft/hcsshim/pkg/securitypolicy/securitypolicyenforcer.go @@ -3,11 +3,17 @@ package securitypolicy import ( "errors" "fmt" + "os" + "path/filepath" "regexp" "strconv" + "strings" "sync" + "github.com/Microsoft/hcsshim/internal/hooks" + "github.com/Microsoft/hcsshim/pkg/annotations" "github.com/google/go-cmp/cmp" + oci "github.com/opencontainers/runtime-spec/specs-go" ) type SecurityPolicyEnforcer interface { @@ -15,6 +21,7 @@ type SecurityPolicyEnforcer interface { EnforceDeviceUnmountPolicy(unmountTarget string) (err error) EnforceOverlayMountPolicy(containerID string, layerPaths []string) (err error) EnforceCreateContainerPolicy(containerID string, argList []string, envList []string, workingDir string) (err error) + EnforceExpectedMountsPolicy(containerID string, spec *oci.Spec) error } func NewSecurityPolicyEnforcer(state SecurityPolicyState) (SecurityPolicyEnforcer, error) { @@ -160,13 +167,19 @@ func (c Container) toInternal() (securityPolicyContainer, error) { return securityPolicyContainer{}, err } + expectedMounts, err := c.ExpectedMounts.toInternal() + if err != nil { + return securityPolicyContainer{}, err + } + return securityPolicyContainer{ Command: command, EnvRules: envRules, Layers: layers, // No need to have toInternal(), because WorkingDir is a string both // internally and in the policy. - WorkingDir: c.WorkingDir, + WorkingDir: c.WorkingDir, + ExpectedMounts: expectedMounts, }, nil } @@ -205,6 +218,14 @@ func (l Layers) toInternal() ([]string, error) { return stringMapToStringArray(l.Elements), nil } +func (em *ExpectedMounts) toInternal() ([]string, error) { + if em.Length != len(em.Elements) { + return nil, fmt.Errorf("expectedMounts numbers don't match in policy. expected: %d, actual: %d", em.Length, len(em.Elements)) + } + + return stringMapToStringArray(em.Elements), nil +} + func stringMapToStringArray(in map[string]string) []string { inLength := len(in) out := make([]string, inLength) @@ -463,7 +484,7 @@ func equalForOverlay(a1 []string, a2 []string) bool { } func possibleIndicesForID(containerID string, mapping map[int]map[string]struct{}) []int { - possibles := []int{} + var possibles []int for index, ids := range mapping { for id := range ids { if containerID == id { @@ -475,6 +496,92 @@ func possibleIndicesForID(containerID string, mapping map[int]map[string]struct{ return possibles } +// EnforceExpectedMountsPolicy for StandardSecurityPolicyEnforcer injects a +// hooks.CreateRuntime hook into container spec and the hook ensures that +// the expected mounts appear prior container start. At the moment enforcement +// is expected to take place inside LCOW UVM. +// +// Expected mount is provided as a path under a sandbox mount path inside +// container, e.g., sandbox mount is at path "/path/in/container" and wait path +// is "/path/in/container/wait/path", which corresponds to +// "/run/gcs/c//sandboxMounts/path/on/the/host/wait/path" +// +// Iterates through container mounts to identify the correct sandbox +// mount where the wait path is nested under. The mount spec will +// be something like: +// { +// "source": "/run/gcs/c//sandboxMounts/path/on/host", +// "destination": "/path/in/container" +// } +// The wait path will be "/path/in/container/wait/path". To find the corresponding +// sandbox mount do a prefix match on wait path against all container mounts +// Destination and resolve the full path inside UVM. For example above it becomes +// "/run/gcs/c//sandboxMounts/path/on/host/wait/path" +func (pe *StandardSecurityPolicyEnforcer) EnforceExpectedMountsPolicy(containerID string, spec *oci.Spec) error { + pe.mutex.Lock() + defer pe.mutex.Unlock() + + if len(pe.Containers) < 1 { + return errors.New("policy doesn't allow mounting containers") + } + + sandboxID := spec.Annotations[annotations.KubernetesSandboxID] + if sandboxID == "" { + return errors.New("no sandbox ID present in spec annotations") + } + + var wMounts []string + pIndices := possibleIndicesForID(containerID, pe.ContainerIndexToContainerIds) + if len(pIndices) == 0 { + return errors.New("no valid container indices found") + } + + // Unlike environment variable and command line enforcement, there isn't anything + // to validate here, since we're essentially just injecting hooks when necessary + // for all containers. + matchFound := false + for _, index := range pIndices { + if !matchFound { + matchFound = true + wMounts = pe.Containers[index].ExpectedMounts + } else { + pe.narrowMatchesForContainerIndex(index, containerID) + } + } + + if len(wMounts) == 0 { + return nil + } + + var wPaths []string + for _, mount := range wMounts { + var wp string + for _, m := range spec.Mounts { + // prefix matching to find correct sandbox mount + if strings.HasPrefix(mount, m.Destination) { + wp = filepath.Join(m.Source, strings.TrimPrefix(mount, m.Destination)) + break + } + } + if wp == "" { + return fmt.Errorf("invalid mount path: %q", mount) + } + wPaths = append(wPaths, filepath.Clean(wp)) + } + + pathsArg := strings.Join(wPaths, ",") + waitPathsBinary := "/bin/wait-paths" + args := []string{ + waitPathsBinary, + "--paths", + pathsArg, + "--timeout", + "60", + } + hook := hooks.NewOCIHook(waitPathsBinary, args, os.Environ()) + return hooks.AddOCIHook(spec, hooks.CreateRuntime, hook) +} + type OpenDoorSecurityPolicyEnforcer struct{} var _ SecurityPolicyEnforcer = (*OpenDoorSecurityPolicyEnforcer)(nil) @@ -495,6 +602,10 @@ func (p *OpenDoorSecurityPolicyEnforcer) EnforceCreateContainerPolicy(_ string, return nil } +func (p *OpenDoorSecurityPolicyEnforcer) EnforceExpectedMountsPolicy(_ string, _ *oci.Spec) error { + return nil +} + type ClosedDoorSecurityPolicyEnforcer struct{} var _ SecurityPolicyEnforcer = (*ClosedDoorSecurityPolicyEnforcer)(nil) @@ -514,3 +625,7 @@ func (p *ClosedDoorSecurityPolicyEnforcer) EnforceOverlayMountPolicy(containerID func (p *ClosedDoorSecurityPolicyEnforcer) EnforceCreateContainerPolicy(_ string, _ []string, _ []string, _ string) (err error) { return errors.New("running commands is denied by policy") } + +func (p *ClosedDoorSecurityPolicyEnforcer) EnforceExpectedMountsPolicy(_ string, _ *oci.Spec) error { + return errors.New("enforcing expected mounts is denied by policy") +} diff --git a/test/vendor/modules.txt b/test/vendor/modules.txt index 496e3d05fe..201ced4412 100644 --- a/test/vendor/modules.txt +++ b/test/vendor/modules.txt @@ -28,6 +28,7 @@ github.com/Microsoft/hcsshim/internal/credentials github.com/Microsoft/hcsshim/internal/devices github.com/Microsoft/hcsshim/internal/extendedtask github.com/Microsoft/hcsshim/internal/gcs +github.com/Microsoft/hcsshim/internal/guestpath github.com/Microsoft/hcsshim/internal/hcs github.com/Microsoft/hcsshim/internal/hcs/resourcepaths github.com/Microsoft/hcsshim/internal/hcs/schema1 @@ -35,6 +36,7 @@ github.com/Microsoft/hcsshim/internal/hcs/schema2 github.com/Microsoft/hcsshim/internal/hcserror github.com/Microsoft/hcsshim/internal/hcsoci github.com/Microsoft/hcsshim/internal/hns +github.com/Microsoft/hcsshim/internal/hooks github.com/Microsoft/hcsshim/internal/interop github.com/Microsoft/hcsshim/internal/layers github.com/Microsoft/hcsshim/internal/lcow