diff --git a/test/functional/lcow_container_bench_test.go b/test/functional/lcow_container_bench_test.go index d2b02838c9..26327fceea 100644 --- a/test/functional/lcow_container_bench_test.go +++ b/test/functional/lcow_container_bench_test.go @@ -4,7 +4,6 @@ package functional import ( - "context" "testing" ctrdoci "github.com/containerd/containerd/oci" @@ -15,7 +14,6 @@ import ( "github.com/Microsoft/hcsshim/osversion" "github.com/Microsoft/hcsshim/test/internal/cmd" - "github.com/Microsoft/hcsshim/test/internal/constants" "github.com/Microsoft/hcsshim/test/internal/container" "github.com/Microsoft/hcsshim/test/internal/layers" "github.com/Microsoft/hcsshim/test/internal/oci" @@ -27,9 +25,8 @@ func BenchmarkLCOW_Container(b *testing.B) { requireFeatures(b, featureLCOW, featureContainer) require.Build(b, osversion.RS5) - ctx, _, client := newContainerdClient(context.Background(), b) - ls := layers.FromImage(ctx, b, client, constants.ImageLinuxAlpineLatest, - constants.PlatformLinux, constants.SnapshotterLinux) + ctx := namespacedContext() + ls := linuxImageLayers(ctx, b) // Create a new uvm per benchmark in case any left over state lingers diff --git a/test/functional/lcow_container_test.go b/test/functional/lcow_container_test.go index 452705ce9c..d8b3372098 100644 --- a/test/functional/lcow_container_test.go +++ b/test/functional/lcow_container_test.go @@ -4,7 +4,6 @@ package functional import ( - "context" "strings" "testing" @@ -13,7 +12,6 @@ import ( "github.com/Microsoft/hcsshim/osversion" "github.com/Microsoft/hcsshim/test/internal/cmd" - "github.com/Microsoft/hcsshim/test/internal/constants" "github.com/Microsoft/hcsshim/test/internal/container" "github.com/Microsoft/hcsshim/test/internal/layers" "github.com/Microsoft/hcsshim/test/internal/oci" @@ -25,10 +23,8 @@ func TestLCOW_ContainerLifecycle(t *testing.T) { requireFeatures(t, featureLCOW, featureContainer) require.Build(t, osversion.RS5) - ctx, _, client := newContainerdClient(context.Background(), t) - ls := layers.FromImage(ctx, t, client, constants.ImageLinuxAlpineLatest, - constants.PlatformLinux, constants.SnapshotterLinux) - + ctx := namespacedContext() + ls := linuxImageLayers(ctx, t) opts := defaultLCOWOptions(t) vm := uvm.CreateAndStartLCOWFromOpts(ctx, t, opts) @@ -81,10 +77,8 @@ func TestLCOW_ContainerIO(t *testing.T) { requireFeatures(t, featureLCOW, featureContainer) require.Build(t, osversion.RS5) - ctx, _, client := newContainerdClient(context.Background(), t) - ls := layers.FromImage(ctx, t, client, constants.ImageLinuxAlpineLatest, - constants.PlatformLinux, constants.SnapshotterLinux) - + ctx := namespacedContext() + ls := linuxImageLayers(ctx, t) opts := defaultLCOWOptions(t) cache := layers.CacheFile(ctx, t, "") vm := uvm.CreateAndStartLCOWFromOpts(ctx, t, opts) @@ -125,10 +119,8 @@ func TestLCOW_ContainerExec(t *testing.T) { requireFeatures(t, featureLCOW, featureContainer) require.Build(t, osversion.RS5) - ctx, _, client := newContainerdClient(context.Background(), t) - ls := layers.FromImage(ctx, t, client, constants.ImageLinuxAlpineLatest, - constants.PlatformLinux, constants.SnapshotterLinux) - + ctx := namespacedContext() + ls := linuxImageLayers(ctx, t) opts := defaultLCOWOptions(t) vm := uvm.CreateAndStartLCOWFromOpts(ctx, t, opts) diff --git a/test/functional/lcow_networking_test.go b/test/functional/lcow_networking_test.go index 1e55c095a6..608831ddfb 100644 --- a/test/functional/lcow_networking_test.go +++ b/test/functional/lcow_networking_test.go @@ -4,7 +4,6 @@ package functional import ( - "context" "fmt" "strings" "testing" @@ -15,7 +14,6 @@ import ( "github.com/Microsoft/hcsshim/osversion" "github.com/Microsoft/hcsshim/test/internal/cmd" - "github.com/Microsoft/hcsshim/test/internal/constants" "github.com/Microsoft/hcsshim/test/internal/container" "github.com/Microsoft/hcsshim/test/internal/layers" "github.com/Microsoft/hcsshim/test/internal/oci" @@ -123,10 +121,8 @@ func TestLCOW_IPv6_Assignment(t *testing.T) { t.Fatalf("network attachment: %v", err) } - ctx, _, client := newContainerdClient(context.Background(), t) - ls := layers.FromImage(ctx, t, client, constants.ImageLinuxAlpineLatest, - constants.PlatformLinux, constants.SnapshotterLinux) - + ctx := namespacedContext() + ls := linuxImageLayers(ctx, t) opts := defaultLCOWOptions(t) vm := uvm.CreateAndStartLCOWFromOpts(ctx, t, opts) diff --git a/test/functional/lcow_test.go b/test/functional/lcow_test.go index f7b178da55..12cca2dbcf 100644 --- a/test/functional/lcow_test.go +++ b/test/functional/lcow_test.go @@ -24,7 +24,6 @@ import ( testutilities "github.com/Microsoft/hcsshim/test/internal" testcmd "github.com/Microsoft/hcsshim/test/internal/cmd" - "github.com/Microsoft/hcsshim/test/internal/layers" "github.com/Microsoft/hcsshim/test/internal/require" testuvm "github.com/Microsoft/hcsshim/test/internal/uvm" ) @@ -165,8 +164,7 @@ func TestLCOWSimplePodScenario(t *testing.T) { require.Build(t, osversion.RS5) requireFeatures(t, featureLCOW, featureContainer) - //nolint:staticcheck // SA1019: TODO: replace `LayerFolders` - alpineLayers := layers.LayerFolders(t, "alpine") + layers := linuxImageLayers(context.Background(), t) cacheDir := t.TempDir() cacheFile := filepath.Join(cacheDir, "cache.vhdx") @@ -201,7 +199,7 @@ func TestLCOWSimplePodScenario(t *testing.T) { t.Fatal(err) } c1Spec := testutilities.GetDefaultLinuxSpec(t) - c1Folders := append(alpineLayers, c1ScratchDir) + c1Folders := append(layers, c1ScratchDir) c1Spec.Windows.LayerFolders = c1Folders c1Spec.Process.Args = []string{"echo", "hello", "lcow", "container", "one"} c1Opts := &hcsoci.CreateOptions{ @@ -214,7 +212,7 @@ func TestLCOWSimplePodScenario(t *testing.T) { t.Fatal(err) } c2Spec := testutilities.GetDefaultLinuxSpec(t) - c2Folders := append(alpineLayers, c2ScratchDir) + c2Folders := append(layers, c2ScratchDir) c2Spec.Windows.LayerFolders = c2Folders c2Spec.Process.Args = []string{"echo", "hello", "lcow", "container", "two"} c2Opts := &hcsoci.CreateOptions{ diff --git a/test/functional/main_test.go b/test/functional/main_test.go index e46cf80fe9..4b7d2d36d2 100644 --- a/test/functional/main_test.go +++ b/test/functional/main_test.go @@ -13,13 +13,12 @@ import ( "log" "os" "os/exec" - "regexp" "strconv" "strings" "testing" "time" - "github.com/containerd/containerd" + "github.com/containerd/containerd/namespaces" "github.com/sirupsen/logrus" "github.com/Microsoft/hcsshim/internal/cow" @@ -28,15 +27,34 @@ import ( "github.com/Microsoft/hcsshim/internal/uvm" "github.com/Microsoft/hcsshim/internal/winapi" - testctrd "github.com/Microsoft/hcsshim/test/internal/containerd" + "github.com/Microsoft/hcsshim/test/internal/constants" testflag "github.com/Microsoft/hcsshim/test/internal/flag" + "github.com/Microsoft/hcsshim/test/internal/layers" "github.com/Microsoft/hcsshim/test/internal/require" + "github.com/Microsoft/hcsshim/test/internal/util" testuvm "github.com/Microsoft/hcsshim/test/internal/uvm" ) // owner field for uVMs. const hcsOwner = "hcsshim-functional-tests" +var ( + alpineImagePaths = &layers.LazyImageLayers{ + Image: constants.ImageLinuxAlpineLatest, + Platform: constants.PlatformLinux, + } + //TODO: pick appropriate image based on OS build + nanoserverImagePaths = &layers.LazyImageLayers{ + Image: constants.ImageWindowsNanoserverLTSC2022, + Platform: constants.PlatformWindows, + } + // wcow tests originally used busyboxw; cannot find image on docker or mcr + servercoreImagePaths = &layers.LazyImageLayers{ + Image: constants.ImageWindowsServercoreLTSC2022, + Platform: constants.PlatformWindows, + } +) + const ( featureLCOW = "LCOW" featureWCOW = "WCOW" @@ -63,16 +81,25 @@ var allFeatures = []string{ featureVPMEM, } -// todo: use a new containerd namespace and then nuke everything in it - var ( - debug bool - pauseDurationOnCreateContainerFailure time.Duration - - flagFeatures = testflag.NewFeatureFlag(allFeatures) - flagContainerdAddress = flag.String("ctr-address", "tcp://127.0.0.1:2376", "`address` for containerd's GRPC server") - flagContainerdNamespace = flag.String("ctr-namespace", "k8s.io", "containerd `namespace`") - flagLinuxBootFilesPath = flag.String("linux-bootfiles", "", + flagPauseAfterCreateContainerFailure time.Duration + + flagFeatures = testflag.NewFeatureFlag(allFeatures) + flagDebug = flag.Bool("debug", + os.Getenv("HCSSHIM_FUNCTIONAL_TESTS_DEBUG") != "", + "set logging level to debug [%HCSSHIM_FUNCTIONAL_TESTS_DEBUG%]") + flagContainerdNamespace = flag.String("ctr-namespace", hcsOwner, + "containerd `namespace` to use when creating OCI specs") + flagLCOWLayerPaths = testflag.NewStringSlice("lcow-layer-paths", + "comma separated list of image layer `paths` to use as LCOW container rootfs. "+ + "If empty, \""+alpineImagePaths.Image+"\" will be pulled and unpacked.") + //nolint:unused // will be used when WCOW tests are updated + flagWCOWLayerPaths = testflag.NewStringSlice("wcow-layer-paths", + "comma separated list of image layer `paths` to use as WCOW uVM and container rootfs. "+ + "If empty, \""+nanoserverImagePaths.Image+"\" will be pulled and unpacked.") + flagLayerTempDir = flag.String("layer-temp-dir", "", + "`directory` to unpack image layers to, if not provided. Leave empty to use os.TempDir.") + flagLinuxBootFilesPath = flag.String("linux-bootfiles", "", "override default `path` for LCOW uVM boot files (rootfs.vhd, initrd.img, kernel, and vmlinux)") ) @@ -81,20 +108,15 @@ func init() { log.Fatal("tests must be run in an elevated context") } - if _, ok := os.LookupEnv("HCSSHIM_FUNCTIONAL_TESTS_DEBUG"); ok { - debug = true - } - flag.BoolVar(&debug, "debug", debug, "set logging level to debug [%HCSSHIM_FUNCTIONAL_TESTS_DEBUG%]") - // This allows for debugging a utility VM. if s := os.Getenv("HCSSHIM_FUNCTIONAL_TESTS_PAUSE_ON_CREATECONTAINER_FAIL_IN_MINUTES"); s != "" { if t, err := strconv.Atoi(s); err == nil { - pauseDurationOnCreateContainerFailure = time.Duration(t) * time.Minute + flagPauseAfterCreateContainerFailure = time.Duration(t) * time.Minute } } - flag.DurationVar(&pauseDurationOnCreateContainerFailure, + flag.DurationVar(&flagPauseAfterCreateContainerFailure, "container-creation-failure-pause", - pauseDurationOnCreateContainerFailure, + flagPauseAfterCreateContainerFailure, "the number of minutes to wait after a container creation failure to try again "+ "[%HCSSHIM_FUNCTIONAL_TESTS_PAUSE_ON_CREATECONTAINER_FAIL_IN_MINUTES%]") } @@ -103,18 +125,23 @@ func TestMain(m *testing.M) { flag.Parse() lvl := logrus.WarnLevel - if debug { + if *flagDebug { lvl = logrus.DebugLevel } logrus.SetLevel(lvl) logrus.SetFormatter(&logrus.TextFormatter{FullTimestamp: true}) logrus.Infof("using features %q", flagFeatures.S.Strings()) + images := []*layers.LazyImageLayers{alpineImagePaths, nanoserverImagePaths, servercoreImagePaths} + for _, l := range images { + l.TempPath = *flagLayerTempDir + } + e := m.Run() // close any uVMs that escaped - cmdStr := `foreach ($vm in Get-ComputeProcess -Owner '` + hcsOwner + `') ` + - `{ Write-Output $vm.Id ; Stop-ComputeProcess -Force -Id $vm.Id }` + cmdStr := ` foreach ($vm in Get-ComputeProcess -Owner '` + hcsOwner + + `') { Write-Output "uVM $($vm.Id) was left running" ; Stop-ComputeProcess -Force -Id $vm.Id } ` cmd := exec.Command("powershell.exe", "-NoLogo", " -NonInteractive", "-Command", cmdStr) o, err := cmd.CombinedOutput() s := string(o) @@ -124,17 +151,23 @@ func TestMain(m *testing.M) { logrus.Warningf("cleaned up left over uVMs: %s", strings.Split(s, "\r\n")) } + // delete downloaded layers; cant use defer, since os.exit does not run them + for _, l := range images { + // just ignore errors: they are logged, and no other cleanup possible + _ = l.Close(context.Background()) + } + os.Exit(e) } func CreateContainerTestWrapper(ctx context.Context, options *hcsoci.CreateOptions) (cow.Container, *resources.Resources, error) { - if pauseDurationOnCreateContainerFailure != 0 { + if flagPauseAfterCreateContainerFailure != 0 { options.DoNotReleaseResourcesOnFailure = true } s, r, err := hcsoci.CreateContainer(ctx, options) if err != nil { - logrus.Warnf("Test is pausing for %s for debugging CreateContainer failure", pauseDurationOnCreateContainerFailure) - time.Sleep(pauseDurationOnCreateContainerFailure) + logrus.Warnf("Test is pausing for %s for debugging CreateContainer failure", flagPauseAfterCreateContainerFailure) + time.Sleep(flagPauseAfterCreateContainerFailure) _ = resources.ReleaseResources(ctx, r, options.HostingSystem, true) } @@ -146,36 +179,57 @@ func requireFeatures(tb testing.TB, features ...string) { require.Features(tb, flagFeatures.S, features...) } -func getContainerdOptions() testctrd.ContainerdClientOptions { - return testctrd.ContainerdClientOptions{ - Address: *flagContainerdAddress, - Namespace: *flagContainerdNamespace, +func defaultLCOWOptions(tb testing.TB) *uvm.OptionsLCOW { + tb.Helper() + opts := testuvm.DefaultLCOWOptions(tb, util.CleanName(tb.Name()), hcsOwner) + if p := *flagLinuxBootFilesPath; p != "" { + opts.BootFilesPath = p } + return opts } -func newContainerdClient(ctx context.Context, tb testing.TB) (context.Context, context.CancelFunc, *containerd.Client) { +//nolint:deadcode,unused // will be used when WCOW tests are updated +func defaultWCOWOptions(tb testing.TB) *uvm.OptionsWCOW { tb.Helper() - return getContainerdOptions().NewClient(ctx, tb) + return uvm.NewDefaultOptionsWCOW(util.CleanName(tb.Name()), hcsOwner) } -func defaultLCOWOptions(tb testing.TB) *uvm.OptionsLCOW { +// linuxImageLayers returns image layer paths appropriate for use as a container rootfs. +// If layer paths were provided on the command line, they are returned. +// Otherwise, it pulls an appropriate image. +func linuxImageLayers(ctx context.Context, tb testing.TB) []string { tb.Helper() - opts := testuvm.DefaultLCOWOptions(tb, cleanName(tb.Name()), hcsOwner) - if p := *flagLinuxBootFilesPath; p != "" { - opts.BootFilesPath = p + if ss := flagLCOWLayerPaths.S.Strings(); len(ss) > 0 { + return ss } - return opts + return alpineImagePaths.Layers(ctx, tb) } +// windowsImageLayers returns image layer paths appropriate for use as a uVM or container rootfs. +// If layer paths were provided on the command line, they are returned. +// Otherwise, it pulls an appropriate image. +// //nolint:deadcode,unused // will be used when WCOW tests are updated -func defaultWCOWOptions(tb testing.TB) *uvm.OptionsWCOW { +func windowsImageLayers(ctx context.Context, tb testing.TB) []string { tb.Helper() - opts := uvm.NewDefaultOptionsWCOW(cleanName(tb.Name()), hcsOwner) - return opts + if ss := flagWCOWLayerPaths.S.Strings(); len(ss) > 0 { + return ss + } + return nanoserverImagePaths.Layers(ctx, tb) } -var _nameRegex = regexp.MustCompile(`[\\\/\s]`) +// windowsServercoreImageLayers returns image layer paths for Windows servercore. +// +// See [windowsImageLayers] for more. +// +//nolint:unused // will be used when WCOW tests are updated +func windowsServercoreImageLayers(ctx context.Context, tb testing.TB) []string { + tb.Helper() + return servercoreImagePaths.Layers(ctx, tb) +} -func cleanName(n string) string { - return _nameRegex.ReplaceAllString(n, "") +// namespacedContext returns a [context.Context] with the provided namespace added via +// [github.com/containerd/containerd/namespaces.WithNamespace]. +func namespacedContext() context.Context { + return namespaces.WithNamespace(context.Background(), *flagContainerdNamespace) } diff --git a/test/functional/uvm_vpmem_test.go b/test/functional/uvm_vpmem_test.go index 14f9a009d4..efc3303010 100644 --- a/test/functional/uvm_vpmem_test.go +++ b/test/functional/uvm_vpmem_test.go @@ -12,7 +12,6 @@ import ( "github.com/Microsoft/hcsshim/internal/copyfile" "github.com/Microsoft/hcsshim/internal/uvm" "github.com/Microsoft/hcsshim/osversion" - "github.com/Microsoft/hcsshim/test/internal/layers" "github.com/Microsoft/hcsshim/test/internal/require" tuvm "github.com/Microsoft/hcsshim/test/internal/uvm" ) @@ -24,10 +23,8 @@ func TestVPMEM(t *testing.T) { require.Build(t, osversion.RS5) requireFeatures(t, featureLCOW, featureVPMEM) - //nolint:staticcheck // SA1019: TODO: replace `LayerFolders` - alpineLayers := layers.LayerFolders(t, "alpine") - ctx := context.Background() + layers := linuxImageLayers(ctx, t) u := tuvm.CreateAndStartLCOW(ctx, t, t.Name()) defer u.Close() @@ -35,7 +32,7 @@ func TestVPMEM(t *testing.T) { // Use layer.vhd from the alpine image as something to add tempDir := t.TempDir() - if err := copyfile.CopyFile(ctx, filepath.Join(alpineLayers[0], "layer.vhd"), filepath.Join(tempDir, "layer.vhd"), true); err != nil { + if err := copyfile.CopyFile(ctx, filepath.Join(layers[0], "layer.vhd"), filepath.Join(tempDir, "layer.vhd"), true); err != nil { t.Fatal(err) } diff --git a/test/functional/wcow_test.go b/test/functional/wcow_test.go index 52ea51a76d..e0fbb97e04 100644 --- a/test/functional/wcow_test.go +++ b/test/functional/wcow_test.go @@ -12,6 +12,8 @@ import ( "strings" "testing" + specs "github.com/opencontainers/runtime-spec/specs-go" + "github.com/Microsoft/hcsshim" "github.com/Microsoft/hcsshim/internal/cow" "github.com/Microsoft/hcsshim/internal/hcs/schema1" @@ -24,9 +26,8 @@ import ( "github.com/Microsoft/hcsshim/internal/wclayer" "github.com/Microsoft/hcsshim/internal/wcow" "github.com/Microsoft/hcsshim/osversion" - "github.com/Microsoft/hcsshim/test/internal/layers" + "github.com/Microsoft/hcsshim/test/internal/require" - specs "github.com/opencontainers/runtime-spec/specs-go" ) // Has testing for Windows containers using both the older hcsshim methods, @@ -347,10 +348,6 @@ func runHcsCommand(t *testing.T, } } -// busybox is used as it has lots of layers. Exercises more code. -// Also the commands are more flexible for verification -const imageName = "busyboxw" - // Creates two temp folders used for the mounts/mapped directories // //nolint:unused // unused since tests are skipped @@ -385,8 +382,7 @@ func TestWCOWArgonShim(t *testing.T) { requireFeatures(t, featureWCOW) - //nolint:staticcheck // SA1019: TODO: replace `LayerFolders` - imageLayers := layers.LayerFolders(t, imageName) + imageLayers := windowsServercoreImageLayers(context.Background(), t) argonShimMounted := false argonShimScratchDir := t.TempDir() @@ -463,8 +459,7 @@ func TestWCOWXenonShim(t *testing.T) { requireFeatures(t, featureWCOW) - //nolint:staticcheck // SA1019: TODO: replace `LayerFolders` - imageLayers := layers.LayerFolders(t, imageName) + imageLayers := windowsServercoreImageLayers(context.Background(), t) xenonShimScratchDir := t.TempDir() if err := wclayer.CreateScratchLayer(context.Background(), xenonShimScratchDir, imageLayers); err != nil { @@ -536,8 +531,7 @@ func TestWCOWArgonOciV1(t *testing.T) { requireFeatures(t, featureWCOW) - //nolint:staticcheck // SA1019: TODO: replace `LayerFolders` - imageLayers := layers.LayerFolders(t, imageName) + imageLayers := windowsServercoreImageLayers(context.Background(), t) argonOci1Mounted := false argonOci1ScratchDir := t.TempDir() if err := wclayer.CreateScratchLayer(context.Background(), argonOci1ScratchDir, imageLayers); err != nil { @@ -585,8 +579,7 @@ func TestWCOWXenonOciV1(t *testing.T) { requireFeatures(t, featureWCOW) - //nolint:staticcheck // SA1019: TODO: replace `LayerFolders` - imageLayers := layers.LayerFolders(t, imageName) + imageLayers := windowsServercoreImageLayers(context.Background(), t) xenonOci1Mounted := false xenonOci1ScratchDir := t.TempDir() @@ -643,8 +636,7 @@ func TestWCOWArgonOciV2(t *testing.T) { require.Build(t, osversion.RS5) requireFeatures(t, featureWCOW) - //nolint:staticcheck // SA1019: TODO: replace `LayerFolders` - imageLayers := layers.LayerFolders(t, imageName) + imageLayers := windowsServercoreImageLayers(context.Background(), t) argonOci2Mounted := false argonOci2ScratchDir := t.TempDir() @@ -694,8 +686,7 @@ func TestWCOWXenonOciV2(t *testing.T) { require.Build(t, osversion.RS5) requireFeatures(t, featureWCOW) - //nolint:staticcheck // SA1019: TODO: replace `LayerFolders` - imageLayers := layers.LayerFolders(t, imageName) + imageLayers := windowsServercoreImageLayers(context.Background(), t) xenonOci2Mounted := false xenonOci2UVMCreated := false diff --git a/test/go.mod b/test/go.mod index 2eefc037fb..3973e204b4 100644 --- a/test/go.mod +++ b/test/go.mod @@ -11,6 +11,7 @@ require ( github.com/containerd/ttrpc v1.1.0 github.com/containerd/typeurl v1.0.2 github.com/gogo/protobuf v1.3.2 + github.com/google/go-containerregistry v0.11.0 github.com/kevpar/cri v1.11.1-0.20220302210600-4c5c347230b2 github.com/opencontainers/go-digest v1.0.0 github.com/opencontainers/image-spec v1.0.3-0.20220114050600-8b9d41f48198 @@ -34,6 +35,7 @@ require ( github.com/containerd/console v1.0.3 // indirect github.com/containerd/continuity v0.2.2 // indirect github.com/containerd/fifo v1.0.0 // indirect + github.com/containerd/stargz-snapshotter/estargz v0.12.0 // indirect github.com/coreos/go-systemd/v22 v22.3.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/docker/cli v20.10.17+incompatible // indirect @@ -48,7 +50,6 @@ require ( github.com/gogo/googleapis v1.4.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect - github.com/google/go-containerregistry v0.11.0 // indirect github.com/google/uuid v1.3.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect @@ -68,6 +69,7 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect + github.com/vbatts/tar-split v0.11.2 // indirect github.com/vektah/gqlparser/v2 v2.4.5 // indirect github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5 // indirect github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f // indirect diff --git a/test/internal/containerd/containerd.go b/test/internal/containerd/containerd.go index 8b835c7033..3f717bee9a 100644 --- a/test/internal/containerd/containerd.go +++ b/test/internal/containerd/containerd.go @@ -12,7 +12,6 @@ import ( "github.com/containerd/containerd/errdefs" kubeutil "github.com/containerd/containerd/integration/remote/util" "github.com/containerd/containerd/mount" - "github.com/containerd/containerd/namespaces" "github.com/containerd/containerd/platforms" "github.com/containerd/containerd/remotes" "github.com/containerd/containerd/remotes/docker" @@ -43,21 +42,19 @@ type ContainerdClientOptions struct { Namespace string } -// NewClient returns a containerd client, a context with the namespace set, and the -// context's cancel function. The context should be used for containerd operations, and -// cancel function will terminate those operations early. +// NewClient returns a containerd client connected using the specified address and namespace. func (cco ContainerdClientOptions) NewClient( ctx context.Context, tb testing.TB, opts ...containerd.ClientOpt, -) (context.Context, context.CancelFunc, *containerd.Client) { +) *containerd.Client { tb.Helper() - // regular `New` does not work on windows, need to use `WithConn` - cctx, ccancel := context.WithTimeout(ctx, timeout.ConnectTimeout) - defer ccancel() + ctx, cancel := context.WithTimeout(ctx, timeout.ConnectTimeout) + defer cancel() - conn, err := createGRPCConn(cctx, cco.Address) + // regular `New` does not work on windows, need to use `WithConn` + conn, err := createGRPCConn(ctx, cco.Address) if err != nil { tb.Fatalf("failed to dial runtime client: %v", err) } @@ -74,11 +71,7 @@ func (cco ContainerdClientOptions) NewClient( c.Close() }) - ctx = namespaces.WithNamespace(ctx, cco.Namespace) - ctx, cancel := context.WithCancel(ctx) - tb.Cleanup(cancel) - - return ctx, cancel, c + return c } func GetPlatformComparer(tb testing.TB, platform string) platforms.MatchComparer { diff --git a/test/internal/flag/flag.go b/test/internal/flag/flag.go index a4cb2c6b5f..fcd25e4890 100644 --- a/test/internal/flag/flag.go +++ b/test/internal/flag/flag.go @@ -11,18 +11,15 @@ import ( const FeatureFlagName = "feature" func NewFeatureFlag(all []string) *StringSlice { - ff := NewStringSlice() - flag.Var(ff, FeatureFlagName, + return NewStringSlice(FeatureFlagName, "the sets of functionality to test; can be set multiple times, or separated with commas. "+ "Supported features: "+strings.Join(all, ", "), ) - - return ff } // StringSlice is a type to be used with the standard library's flag.Var // function as a custom flag value, similar to "github.com/urfave/cli".StringSlice. -// It takes either a comma-separated list of strings, or repeat invocations. +// It takes either a comma-separated list of strings, or repeated invocations. type StringSlice struct { S StringSet } @@ -30,10 +27,12 @@ type StringSlice struct { var _ flag.Value = &StringSlice{} // NewStringSetFlag returns a new StringSetFlag with an empty set. -func NewStringSlice() *StringSlice { - return &StringSlice{ +func NewStringSlice(name, usage string) *StringSlice { + ss := &StringSlice{ S: make(StringSet), } + flag.Var(ss, name, usage) + return ss } // Strings returns a string slice of the flags provided to the flag diff --git a/test/internal/layers/layerfolders.go b/test/internal/layers/layerfolders.go index be0c506c57..13c6be54df 100644 --- a/test/internal/layers/layerfolders.go +++ b/test/internal/layers/layerfolders.go @@ -3,12 +3,8 @@ package layers import ( - "bytes" "context" "encoding/json" - "os" - "os/exec" - "path/filepath" "strings" "testing" @@ -18,12 +14,6 @@ import ( testctrd "github.com/Microsoft/hcsshim/test/internal/containerd" ) -var imageLayers map[string][]string - -func init() { - imageLayers = make(map[string][]string) -} - // FromImage returns thee layer paths of a given image, pulling it if necessary func FromImage(ctx context.Context, tb testing.TB, client *containerd.Client, ref, platform, snapshotter string) []string { tb.Helper() @@ -57,43 +47,3 @@ func FromMount(_ context.Context, tb testing.TB, m mount.Mount) (layers []string return layers } - -// Deprecated: This relies on docker. Use [FromChainID] or [FromMount] instead. -func LayerFolders(tb testing.TB, imageName string) []string { - tb.Helper() - if _, ok := imageLayers[imageName]; !ok { - imageLayers[imageName] = getLayers(tb, imageName) - } - return imageLayers[imageName] -} - -func getLayers(tb testing.TB, imageName string) []string { - tb.Helper() - cmd := exec.Command("docker", "inspect", imageName, "-f", `"{{.GraphDriver.Data.dir}}"`) - var out bytes.Buffer - cmd.Stdout = &out - if err := cmd.Run(); err != nil { - tb.Skipf("Failed to find layers for %q. Check docker images", imageName) - } - imagePath := strings.Replace(strings.TrimSpace(out.String()), `"`, ``, -1) - layers := getLayerChain(tb, imagePath) - return append([]string{imagePath}, layers...) -} - -func getLayerChain(tb testing.TB, layerFolder string) []string { - tb.Helper() - jPath := filepath.Join(layerFolder, "layerchain.json") - content, err := os.ReadFile(jPath) - if os.IsNotExist(err) { - tb.Fatalf("layerchain not found") - } else if err != nil { - tb.Fatalf("failed to read layerchain") - } - - var layerChain []string - err = json.Unmarshal(content, &layerChain) - if err != nil { - tb.Fatalf("failed to unmarshal layerchain") - } - return layerChain -} diff --git a/test/internal/layers/lazy.go b/test/internal/layers/lazy.go new file mode 100644 index 0000000000..ec4d13c156 --- /dev/null +++ b/test/internal/layers/lazy.go @@ -0,0 +1,167 @@ +//go:build windows + +package layers + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "runtime" + "strconv" + "sync" + "testing" + + "github.com/Microsoft/go-winio" + "github.com/google/go-containerregistry/pkg/crane" + v1 "github.com/google/go-containerregistry/pkg/v1" + + "github.com/Microsoft/hcsshim/ext4/tar2ext4" + "github.com/Microsoft/hcsshim/internal/log" + "github.com/Microsoft/hcsshim/internal/security" + "github.com/Microsoft/hcsshim/internal/wclayer" + "github.com/Microsoft/hcsshim/pkg/ociwclayer" + + "github.com/Microsoft/hcsshim/test/internal/constants" + "github.com/Microsoft/hcsshim/test/internal/util" +) + +// helper utilities for dealing with images + +type LazyImageLayers struct { + Image string + Platform string + // TempPath is the path to create a temporary directory in. + // Defaults to [os.TempDir] if left empty. + TempPath string + // dedicated directory, under [TempPath], to store layers in + dir string + once sync.Once + layers []string +} + +// Close removes the downloaded image layers. +// +// Does not take a [testing.TB] so it can be used in TestMain or init. +func (x *LazyImageLayers) Close(ctx context.Context) error { + if x.dir == "" { + return nil + } + + if _, err := os.Stat(x.dir); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("path %q is not valid: %w", x.dir, err) + } + // DestroyLayer will remove the entire directory and all its contents, regardless of if + // its a Windows container layer or not. + if err := wclayer.DestroyLayer(ctx, x.dir); err != nil { + return fmt.Errorf("could not destroy layer directory %q: %w", x.dir, err) + } + return nil +} + +// Layers returns the image layer paths, from lowest to highest, for a particular image. +func (x *LazyImageLayers) Layers(ctx context.Context, tb testing.TB) []string { + // basically combo of containerd fetch and unpack (snapshotter + differ) + tb.Helper() + var err error + x.once.Do(func() { + err = x.extractLayers(ctx) + }) + if err != nil { + x.Close(ctx) + tb.Fatal(err) + } + return x.layers +} + +// don't use tb.Error/Log inside Once.Do stack, since we cannot call tb.Helper before executing f() +// within Once.Do and that will therefore show the wrong stack/location +func (x *LazyImageLayers) extractLayers(ctx context.Context) (err error) { + log.G(ctx).Infof("pulling and unpacking %s image %q", x.Platform, x.Image) + + if x.TempPath == "" { + dir := os.TempDir() + x.dir, err = os.MkdirTemp(dir, util.CleanName(x.Image)) + if err != nil { + return fmt.Errorf("failed to create temp directory: %w", err) + } + } else { + x.dir, err = filepath.Abs(x.TempPath) + if err != nil { + return fmt.Errorf("failed to make %q absolute path: %w", x.TempPath, err) + } + } + + var extract func(context.Context, io.ReadCloser, string, []string) error + switch x.Platform { + case constants.PlatformLinux: + extract = linuxImage + case constants.PlatformWindows: + if err = winio.EnableProcessPrivileges([]string{winio.SeBackupPrivilege, winio.SeRestorePrivilege}); err != nil { + return err + } + extract = windowsImage + default: + return fmt.Errorf("unsupported platform %q", x.Platform) + } + + img, err := crane.Pull(x.Image, crane.WithPlatform(&v1.Platform{OS: x.Platform, Architecture: runtime.GOARCH})) + if err != nil { + return fmt.Errorf("failed to pull image %q: %w", x.Image, err) + } + + layers, err := img.Layers() + if err != nil { + return fmt.Errorf("failed to get image %q layers: %w", x.Image, err) + } + + for i, l := range layers { + d := filepath.Join(x.dir, strconv.FormatInt(int64(i), 10)) + if err := os.Mkdir(d, 0755); err != nil { + return err + } + rc, err := l.Uncompressed() + if err != nil { + return fmt.Errorf("failed to load uncompressed layer for image %s: %w", x.Image, err) + } + defer rc.Close() + if err := extract(ctx, rc, d, x.layers); err != nil { + return fmt.Errorf("failed to extract layer %d for image %s: %w", i, x.Image, err) + } + x.layers = append(x.layers, d) + } + + return nil +} + +func linuxImage(ctx context.Context, rc io.ReadCloser, dir string, _ []string) error { + f, err := os.Create(filepath.Join(dir, "layer.vhd")) + if err != nil { + return fmt.Errorf("create layer vhd: %w", err) + } + // in case we fail before granting access; double close on file will no-op + defer f.Close() + + if err := tar2ext4.Convert(rc, f, tar2ext4.AppendVhdFooter, tar2ext4.ConvertWhiteout); err != nil { + return fmt.Errorf("convert to vhd %s: %w", f.Name(), err) + } + if err := f.Sync(); err != nil { + return fmt.Errorf("sync vhd %s to disk: %w", f.Name(), err) + } + f.Close() + + if err = security.GrantVmGroupAccess(f.Name()); err != nil { + return fmt.Errorf("grant vm group access to %s: %w", f.Name(), err) + } + + return nil +} + +func windowsImage(ctx context.Context, rc io.ReadCloser, dir string, parents []string) error { + if _, err := ociwclayer.ImportLayerFromTar(ctx, rc, dir, parents); err != nil { + return fmt.Errorf("import wc layer %s: %w", dir, err) + } + + return nil +} diff --git a/test/internal/oci/oci.go b/test/internal/oci/oci.go index 5369bda298..25d1a74983 100644 --- a/test/internal/oci/oci.go +++ b/test/internal/oci/oci.go @@ -43,28 +43,32 @@ func DefaultLinuxSpecOpts(nns string, extra ...ctrdoci.SpecOpts) []ctrdoci.SpecO } // DefaultLinuxSpec returns a default OCI spec for a Linux container. -// See CreateSpecWithPlatform for more details. +// +// See [CreateSpecWithPlatform] for more details. func DefaultLinuxSpec(ctx context.Context, tb testing.TB, nns string) *specs.Spec { tb.Helper() return CreateLinuxSpec(ctx, tb, tb.Name(), DefaultLinuxSpecOpts(nns)...) } // CreateLinuxSpec returns the OCI spec for a Linux container. -// See CreateSpecWithPlatform for more details. +// +// See [CreateSpecWithPlatform] for more details. func CreateLinuxSpec(ctx context.Context, tb testing.TB, id string, opts ...ctrdoci.SpecOpts) *specs.Spec { tb.Helper() return CreateSpecWithPlatform(ctx, tb, constants.PlatformLinux, id, opts...) } // CreateWindowsSpec returns the OCI spec for a Windows container. -// See CreateSpecWithPlatform for more details. +// +// See [CreateSpecWithPlatform] for more details. func CreateWindowsSpec(ctx context.Context, tb testing.TB, id string, opts ...ctrdoci.SpecOpts) *specs.Spec { tb.Helper() return CreateSpecWithPlatform(ctx, tb, constants.PlatformWindows, id, opts...) } // CreateSpecWithPlatform returns the OCI spec for the specified platform. -// The context must contain a containerd namespace (via "github.com/containerd/containerd/namespaces".WithNamespace) +// The context must contain a containerd namespace added by +// [github.com/containerd/containerd/namespaces.WithNamespace] func CreateSpecWithPlatform(ctx context.Context, tb testing.TB, plat, id string, opts ...ctrdoci.SpecOpts) *specs.Spec { tb.Helper() container := &containers.Container{ID: id} diff --git a/test/internal/util/util.go b/test/internal/util/util.go new file mode 100644 index 0000000000..5f3d15e386 --- /dev/null +++ b/test/internal/util/util.go @@ -0,0 +1,20 @@ +package util + +import ( + "strings" + "unicode" +) + +// CleanName returns a string appropriate for uVM, container, or file names. +// +// Based on [testing.TB.TempDir]. +func CleanName(n string) string { + mapper := func(r rune) rune { + const allowed = "!#$%&()+,-.=@^_{}~ " + if unicode.IsLetter(r) || unicode.IsNumber(r) || strings.ContainsRune(allowed, r) { + return r + } + return -1 + } + return strings.TrimSpace(strings.Map(mapper, n)) +} diff --git a/test/internal/uvm/lcow.go b/test/internal/uvm/lcow.go index 2c8943f6e1..a42d432705 100644 --- a/test/internal/uvm/lcow.go +++ b/test/internal/uvm/lcow.go @@ -45,7 +45,6 @@ func DefaultLCOWOptions(tb testing.TB, id, owner string) *uvm.OptionsLCOW { tb.Helper() opts := uvm.NewDefaultOptionsLCOW(id, owner) if lcowOSBootFiles != "" { - tb.Logf("using LCOW bootfiles path: %s", lcowOSBootFiles) opts.BootFilesPath = lcowOSBootFiles } return opts diff --git a/test/internal/uvm/wcow.go b/test/internal/uvm/wcow.go index daef686d94..f165e7a58d 100644 --- a/test/internal/uvm/wcow.go +++ b/test/internal/uvm/wcow.go @@ -8,6 +8,7 @@ import ( "github.com/Microsoft/hcsshim/internal/uvm" + "github.com/Microsoft/hcsshim/test/internal/constants" "github.com/Microsoft/hcsshim/test/internal/layers" ) @@ -54,8 +55,14 @@ func CreateWCOWUVMFromOptsWithImage( tb.Fatal("opts must be set") } - //nolint:staticcheck // SA1019: TODO: switch from LayerFolders - uvmLayers := layers.LayerFolders(tb, image) + img := layers.LazyImageLayers{Image: image, Platform: constants.PlatformWindows} + tb.Cleanup(func() { + if err := img.Close(ctx); err != nil { + tb.Errorf("could not close image %s: %v", image, err) + } + }) + + uvmLayers := img.Layers(ctx, tb) scratchDir := tb.TempDir() opts.LayerFolders = append(opts.LayerFolders, uvmLayers...) opts.LayerFolders = append(opts.LayerFolders, scratchDir) diff --git a/test/runhcs/e2e_matrix_test.go b/test/runhcs/e2e_matrix_test.go index 2ed1eb8944..d104c279a3 100644 --- a/test/runhcs/e2e_matrix_test.go +++ b/test/runhcs/e2e_matrix_test.go @@ -18,6 +18,7 @@ import ( "github.com/Microsoft/go-winio/vhd" "github.com/Microsoft/hcsshim/osversion" runhcs "github.com/Microsoft/hcsshim/pkg/go-runhcs" + "github.com/Microsoft/hcsshim/test/internal/constants" "github.com/Microsoft/hcsshim/test/internal/layers" "github.com/Microsoft/hcsshim/test/internal/require" runc "github.com/containerd/go-runc" @@ -168,6 +169,7 @@ func readPidFile(path string) (int, error) { func testWindows(t *testing.T, version int, isolated bool) { t.Helper() var err error + ctx := context.Background() // Make the bundle bundle := t.TempDir() @@ -198,8 +200,14 @@ func testWindows(t *testing.T, version int, isolated bool) { // Get the LayerFolders imageName := getWindowsImageNameByVersion(t, version) - //nolint:staticcheck // SA1019: TODO: replace `LayerFolders` - layers := layers.LayerFolders(t, imageName) + img := layers.LazyImageLayers{Image: imageName, Platform: constants.PlatformWindows} + defer func() { + if err := img.Close(ctx); err != nil { + t.Errorf("could not close image %s: %v", imageName, err) + } + }() + + layers := img.Layers(ctx, t) for _, layer := range layers { g.AddWindowsLayerFolders(layer) } @@ -219,7 +227,6 @@ func testWindows(t *testing.T, version int, isolated bool) { cf.Close() // Create the Argon, Xenon, or UVM - ctx := context.TODO() rhcs := runhcs.Runhcs{ Debug: true, }