Skip to content

Commit

Permalink
Add flag for container layer paths
Browse files Browse the repository at this point in the history 
Add lcow/wcow layer paths flag to allow providing image layer paths
instead of requiring containerd to pull and unpack it.

The goal is to allow functional tests to be run without requiring
containerd to be installed.

Signed-off-by: Hamza El-Saawy <hamzaelsaawy@microsoft.com>
  • Loading branch information
@helsaawy
helsaawy committed Oct 6, 2022
1 parent 05b973d commit ff68acc
Show file tree
Hide file tree
Showing 11 changed files with 293 additions and 103 deletions.
7 changes: 2 additions & 5 deletions test/functional/lcow_container_bench_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
package functional

import (
"context"
"testing"

ctrdoci "github.com/containerd/containerd/oci"
Expand All @@ -15,7 +14,6 @@ import (
"github.com/Microsoft/hcsshim/osversion"

"github.com/Microsoft/hcsshim/test/internal/cmd"
"github.com/Microsoft/hcsshim/test/internal/constants"
"github.com/Microsoft/hcsshim/test/internal/container"
"github.com/Microsoft/hcsshim/test/internal/layers"
"github.com/Microsoft/hcsshim/test/internal/oci"
Expand All @@ -27,9 +25,8 @@ func BenchmarkLCOW_Container(b *testing.B) {
requireFeatures(b, featureLCOW, featureContainer)
require.Build(b, osversion.RS5)

ctx, _, client := newContainerdClient(context.Background(), b)
ls := layers.FromImage(ctx, b, client, constants.ImageLinuxAlpineLatest,
constants.PlatformLinux, constants.SnapshotterLinux)
ctx := namespacedContext()
ls := linuxImageLayers(ctx, b)

// Create a new uvm per benchmark in case any left over state lingers

Expand Down
20 changes: 6 additions & 14 deletions test/functional/lcow_container_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
package functional

import (
"context"
"strings"
"testing"

Expand All @@ -13,7 +12,6 @@ import (
"github.com/Microsoft/hcsshim/osversion"

"github.com/Microsoft/hcsshim/test/internal/cmd"
"github.com/Microsoft/hcsshim/test/internal/constants"
"github.com/Microsoft/hcsshim/test/internal/container"
"github.com/Microsoft/hcsshim/test/internal/layers"
"github.com/Microsoft/hcsshim/test/internal/oci"
Expand All @@ -25,10 +23,8 @@ func TestLCOW_ContainerLifecycle(t *testing.T) {
requireFeatures(t, featureLCOW, featureContainer)
require.Build(t, osversion.RS5)

ctx, _, client := newContainerdClient(context.Background(), t)
ls := layers.FromImage(ctx, t, client, constants.ImageLinuxAlpineLatest,
constants.PlatformLinux, constants.SnapshotterLinux)

ctx := namespacedContext()
ls := linuxImageLayers(ctx, t)
opts := defaultLCOWOptions(t)
vm := uvm.CreateAndStartLCOWFromOpts(ctx, t, opts)

Expand Down Expand Up @@ -81,10 +77,8 @@ func TestLCOW_ContainerIO(t *testing.T) {
requireFeatures(t, featureLCOW, featureContainer)
require.Build(t, osversion.RS5)

ctx, _, client := newContainerdClient(context.Background(), t)
ls := layers.FromImage(ctx, t, client, constants.ImageLinuxAlpineLatest,
constants.PlatformLinux, constants.SnapshotterLinux)

ctx := namespacedContext()
ls := linuxImageLayers(ctx, t)
opts := defaultLCOWOptions(t)
cache := layers.CacheFile(ctx, t, "")
vm := uvm.CreateAndStartLCOWFromOpts(ctx, t, opts)
Expand Down Expand Up @@ -125,10 +119,8 @@ func TestLCOW_ContainerExec(t *testing.T) {
requireFeatures(t, featureLCOW, featureContainer)
require.Build(t, osversion.RS5)

ctx, _, client := newContainerdClient(context.Background(), t)
ls := layers.FromImage(ctx, t, client, constants.ImageLinuxAlpineLatest,
constants.PlatformLinux, constants.SnapshotterLinux)

ctx := namespacedContext()
ls := linuxImageLayers(ctx, t)
opts := defaultLCOWOptions(t)
vm := uvm.CreateAndStartLCOWFromOpts(ctx, t, opts)

Expand Down
112 changes: 70 additions & 42 deletions test/functional/main_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,11 @@ import (
"log"
"os"
"os/exec"
"regexp"
"strconv"
"testing"
"time"

"github.com/containerd/containerd"
"github.com/containerd/containerd/namespaces"
"github.com/sirupsen/logrus"

"github.com/Microsoft/hcsshim/internal/cow"
Expand All @@ -27,14 +26,28 @@ import (
"github.com/Microsoft/hcsshim/internal/uvm"
"github.com/Microsoft/hcsshim/internal/winapi"

testctrd "github.com/Microsoft/hcsshim/test/internal/containerd"
"github.com/Microsoft/hcsshim/test/internal/constants"
testflag "github.com/Microsoft/hcsshim/test/internal/flag"
"github.com/Microsoft/hcsshim/test/internal/layers"
"github.com/Microsoft/hcsshim/test/internal/require"
"github.com/Microsoft/hcsshim/test/internal/util"
)

// owner field for uVMs.
const hcsOwner = "hcsshim-functional-tests"

var (
alpineImagePaths = layers.LazyImageLayers{
Image: constants.ImageLinuxAlpineLatest,
Platform: constants.PlatformLinux,
}
//TODO: pick appropriate image based on OS build
nanoserverImagePaths = layers.LazyImageLayers{
Image: constants.ImageWindowsNanoserverLTSC2022,
Platform: constants.PlatformWindows,
}
)

const (
featureLCOW = "LCOW"
featureWCOW = "WCOW"
Expand All @@ -61,16 +74,23 @@ var allFeatures = []string{
featureVPMEM,
}

// todo: use a new containerd namespace and then nuke everything in it

var (
debug bool
pauseDurationOnCreateContainerFailure time.Duration

flagFeatures = testflag.NewFeatureFlag(allFeatures)
flagContainerdAddress = flag.String("ctr-address", "tcp://127.0.0.1:2376", "`address` for containerd's GRPC server")
flagContainerdNamespace = flag.String("ctr-namespace", "k8s.io", "containerd `namespace`")
flagLinuxBootFilesPath = flag.String("linux-bootfiles",
flagPauseAfterCreateContainerFailure time.Duration

flagFeatures = testflag.NewFeatureFlag(allFeatures)
flagDebug = flag.Bool("debug",
os.Getenv("HCSSHIM_FUNCTIONAL_TESTS_DEBUG") != "",
"set logging level to debug [%HCSSHIM_FUNCTIONAL_TESTS_DEBUG%]")
flagContainerdNamespace = flag.String("ctr-namespace", hcsOwner,
"containerd `namespace` to use when creating OCI specs")
flagLCOWLayerPaths = testflag.NewStringSlice("lcow-layer-paths",
"comma separated list of image layer `paths` to use as LCOW container rootfs. "+
"If empty, \""+alpineImagePaths.Image+"\" will be pulled and unpacked.")
//nolint:unused // will be used when WCOW tests are updated
flagWCOWLayerPaths = testflag.NewStringSlice("wcow-layer-paths",
"comma separated list of image layer `paths` to use as WCO as WCOW uVM and container rootfs. "+
"If empty, \""+nanoserverImagePaths.Image+"\" will be pulled and unpacked.")
flagLinuxBootFilesPath = flag.String("linux-bootfiles",
`C:\\ContainerPlat\\LinuxBootFiles`,
"`path` to LCOW UVM boot files (rootfs.vhd, initrd.img, kernel, and vmlinux)")
)
Expand All @@ -80,20 +100,15 @@ func init() {
log.Fatal("tests must be run in an elevated context")
}

if _, ok := os.LookupEnv("HCSSHIM_FUNCTIONAL_TESTS_DEBUG"); ok {
debug = true
}
flag.BoolVar(&debug, "debug", debug, "set logging level to debug [%HCSSHIM_FUNCTIONAL_TESTS_DEBUG%]")

// This allows for debugging a utility VM.
if s := os.Getenv("HCSSHIM_FUNCTIONAL_TESTS_PAUSE_ON_CREATECONTAINER_FAIL_IN_MINUTES"); s != "" {
if t, err := strconv.Atoi(s); err == nil {
pauseDurationOnCreateContainerFailure = time.Duration(t) * time.Minute
flagPauseAfterCreateContainerFailure = time.Duration(t) * time.Minute
}
}
flag.DurationVar(&pauseDurationOnCreateContainerFailure,
flag.DurationVar(&flagPauseAfterCreateContainerFailure,
"container-creation-failure-pause",
pauseDurationOnCreateContainerFailure,
flagPauseAfterCreateContainerFailure,
"the number of minutes to wait after a container creation failure to try again "+
"[%HCSSHIM_FUNCTIONAL_TESTS_PAUSE_ON_CREATECONTAINER_FAIL_IN_MINUTES%]")
}
Expand All @@ -102,19 +117,23 @@ func TestMain(m *testing.M) {
flag.Parse()

lvl := logrus.WarnLevel
if vf := flag.Lookup("test.v"); debug || (vf != nil && vf.Value.String() == strconv.FormatBool(true)) {
if vf := flag.Lookup("test.v"); *flagDebug || (vf != nil && vf.Value.String() == strconv.FormatBool(true)) {
lvl = logrus.DebugLevel
}
logrus.SetLevel(lvl)
logrus.SetFormatter(&logrus.TextFormatter{FullTimestamp: true})
logrus.Infof("using features %q", flagFeatures.S.Strings())

// deleted downloaded layers
defer alpineImagePaths.Close(context.Background())
defer nanoserverImagePaths.Close(context.Background())

e := m.Run()

// close any uVMs that escaped
cmdStr := ` foreach ($vm in Get-ComputeProcess -Owner '` + hcsOwner +
`') { Write-Output "uVM $($vm.Id) was left running" ; Stop-ComputeProcess -Force -Id $vm.Id } `
cmd := exec.Command("powershell", "-NoLogo", " -NonInteractive", "-Command", cmdStr)
cmd := exec.Command("powershell.exe", "-NoLogo", " -NonInteractive", "-Command", cmdStr)
o, err := cmd.CombinedOutput()
if err != nil {
logrus.Warningf("could not call %q to clean up remaining uVMs: %v", cmdStr, err)
Expand All @@ -126,13 +145,13 @@ func TestMain(m *testing.M) {
}

func CreateContainerTestWrapper(ctx context.Context, options *hcsoci.CreateOptions) (cow.Container, *resources.Resources, error) {
if pauseDurationOnCreateContainerFailure != 0 {
if flagPauseAfterCreateContainerFailure != 0 {
options.DoNotReleaseResourcesOnFailure = true
}
s, r, err := hcsoci.CreateContainer(ctx, options)
if err != nil {
logrus.Warnf("Test is pausing for %s for debugging CreateContainer failure", pauseDurationOnCreateContainerFailure)
time.Sleep(pauseDurationOnCreateContainerFailure)
logrus.Warnf("Test is pausing for %s for debugging CreateContainer failure", flagPauseAfterCreateContainerFailure)
time.Sleep(flagPauseAfterCreateContainerFailure)
_ = resources.ReleaseResources(ctx, r, options.HostingSystem, true)
}

Expand All @@ -143,33 +162,42 @@ func requireFeatures(t testing.TB, features ...string) {
require.Features(t, flagFeatures.S, features...)
}

func getContainerdOptions() testctrd.ContainerdClientOptions {
return testctrd.ContainerdClientOptions{
Address: *flagContainerdAddress,
Namespace: *flagContainerdNamespace,
}
}

func newContainerdClient(ctx context.Context, t testing.TB) (context.Context, context.CancelFunc, *containerd.Client) {
return getContainerdOptions().NewClient(ctx, t)
}

func defaultLCOWOptions(t testing.TB) *uvm.OptionsLCOW {
opts := uvm.NewDefaultOptionsLCOW(cleanName(t.Name()), "")
opts := uvm.NewDefaultOptionsLCOW(util.CleanName(t.Name()), hcsOwner)
opts.BootFilesPath = *flagLinuxBootFilesPath

return opts
}

//nolint:deadcode,unused // will be used when WCOW tests are updated
func defaultWCOWOptions(t testing.TB) *uvm.OptionsWCOW {
opts := uvm.NewDefaultOptionsWCOW(cleanName(t.Name()), "")
return uvm.NewDefaultOptionsWCOW(util.CleanName(t.Name()), hcsOwner)
}

return opts
// linuxImageLayers returns image layer paths appropriate for use as a container rootfs.
// If layer paths were provided on the command line, they are returned.
// Otherwise, it pulls an appropriate image.
func linuxImageLayers(ctx context.Context, tb testing.TB) []string {
if ss := flagLCOWLayerPaths.S.Strings(); len(ss) > 0 {
return ss
}
return alpineImagePaths.ImageLayers(ctx, tb)
}

var _nameRegex = regexp.MustCompile(`[\\\/\s]`)
// windowsImageLayers returns image layer paths appropriate for use as a uVM or container rootfs.
// If layer paths were provided on the command line, they are returned.
// Otherwise, it pulls an appropriate image.
//
//nolint:deadcode,unused // will be used when WCOW tests are updated
func windowsImageLayers(ctx context.Context, tb testing.TB) []string {
if ss := flagWCOWLayerPaths.S.Strings(); len(ss) > 0 {
return ss
}
return nanoserverImagePaths.ImageLayers(ctx, tb)
}

func cleanName(n string) string {
return _nameRegex.ReplaceAllString(n, "")
// namespacedContext returns a [context.Context] with the provided namespace added via
// [github.com/containerd/containerd/namespaces.WithNamespace].
func namespacedContext() context.Context {
return namespaces.WithNamespace(context.Background(), *flagContainerdNamespace)
}
21 changes: 6 additions & 15 deletions test/functional/uvm_update_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,12 @@ import (
"context"
"testing"

"github.com/opencontainers/runtime-spec/specs-go"

"github.com/Microsoft/hcsshim/internal/protocol/guestrequest"
"github.com/Microsoft/hcsshim/internal/uvm"
"github.com/Microsoft/hcsshim/pkg/ctrdtaskapi"
"github.com/opencontainers/runtime-spec/specs-go"

"github.com/Microsoft/hcsshim/test/internal/uvm"
)

func Test_LCOW_Update_Resources(t *testing.T) {
Expand Down Expand Up @@ -47,19 +49,8 @@ func Test_LCOW_Update_Resources(t *testing.T) {
} {
t.Run(config.name, func(t *testing.T) {
ctx := context.Background()
opts := uvm.NewDefaultOptionsLCOW(t.Name(), t.Name())
vm, err := uvm.CreateLCOW(ctx, opts)
if err != nil {
t.Fatalf("failed to create LCOW UVM: %s", err)
}
if err := vm.Start(ctx); err != nil {
t.Fatalf("failed to start LCOW UVM: %s", err)
}
t.Cleanup(func() {
if err := vm.Close(); err != nil {
t.Log(err)
}
})
vm := uvm.CreateLCOW(ctx, t, defaultLCOWOptions(t))
uvm.Start(ctx, t, vm)
if err := vm.Update(ctx, config.resource, nil); err != nil {
if config.valid {
t.Fatalf("failed to update LCOW UVM constraints: %s", err)
Expand Down
4 changes: 3 additions & 1 deletion test/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ require (
github.com/containerd/ttrpc v1.1.0
github.com/containerd/typeurl v1.0.2
github.com/gogo/protobuf v1.3.2
github.com/google/go-containerregistry v0.11.0
github.com/kevpar/cri v1.11.1-0.20220302210600-4c5c347230b2
github.com/opencontainers/go-digest v1.0.0
github.com/opencontainers/image-spec v1.0.3-0.20220114050600-8b9d41f48198
Expand All @@ -34,6 +35,7 @@ require (
github.com/containerd/console v1.0.3 // indirect
github.com/containerd/continuity v0.2.2 // indirect
github.com/containerd/fifo v1.0.0 // indirect
github.com/containerd/stargz-snapshotter/estargz v0.12.0 // indirect
github.com/coreos/go-systemd/v22 v22.3.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/docker/cli v20.10.17+incompatible // indirect
Expand All @@ -48,7 +50,6 @@ require (
github.com/gogo/googleapis v1.4.0 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/google/go-containerregistry v0.11.0 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
Expand All @@ -68,6 +69,7 @@ require (
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect
github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect
github.com/vbatts/tar-split v0.11.2 // indirect
github.com/vektah/gqlparser/v2 v2.4.5 // indirect
github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5 // indirect
github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f // indirect
Expand Down
Loading

0 comments on commit ff68acc

Please sign in to comment.