Add exec benchmarks (#1855)

Add benchmarks to measure (LCOW) exec performance to functional test
suite.

Additionally, remove helper testing functions from benchmark section
(between `b.StartTimer()` and `b.StopTimer()`), since helper functions
will call `"testing".(TB).Helper()`, which involves mutex (un)lock
operations as well as parsing stack frames.

Signed-off-by: Hamza El-Saawy <hamzaelsaawy@microsoft.com>

test/functional/lcow_container_bench_test.go

@@ -4,18 +4,21 @@
 package functional
 
 import (
+ "errors"
  "path/filepath"
+ "strings"
  "testing"
 
  ctrdoci "github.com/containerd/containerd/oci"
  cri_util "github.com/containerd/containerd/pkg/cri/util"
 
+ "github.com/Microsoft/hcsshim/internal/cmd"
  "github.com/Microsoft/hcsshim/internal/hcsoci"
  "github.com/Microsoft/hcsshim/internal/layers"
  "github.com/Microsoft/hcsshim/internal/resources"
  "github.com/Microsoft/hcsshim/osversion"
 
- "github.com/Microsoft/hcsshim/test/internal/cmd"
+ testcmd "github.com/Microsoft/hcsshim/test/internal/cmd"
  "github.com/Microsoft/hcsshim/test/internal/container"
  testlayers "github.com/Microsoft/hcsshim/test/internal/layers"
  "github.com/Microsoft/hcsshim/test/internal/oci"
@@ -75,7 +78,7 @@ func BenchmarkLCOW_Container(b *testing.B) {
  // not finish until the init process has terminated.
  // so start the container, then clean everything up
  init := container.Start(ctx, b, c, nil)
- cmd.WaitExitCode(ctx, b, init, 0)
+ testcmd.WaitExitCode(ctx, b, init, 0)
 
  container.Kill(ctx, b, c)
  container.Wait(ctx, b, c)
@@ -110,9 +113,9 @@ func BenchmarkLCOW_Container(b *testing.B) {
  }
  b.StopTimer()
 
- init := cmd.Create(ctx, b, c, nil, nil)
- cmd.Start(ctx, b, init)
- cmd.WaitExitCode(ctx, b, init, 0)
+ init := testcmd.Create(ctx, b, c, nil, nil)
+ testcmd.Start(ctx, b, init)
+ testcmd.WaitExitCode(ctx, b, init, 0)
 
  container.Kill(ctx, b, c)
  container.Wait(ctx, b, c)
@@ -138,14 +141,16 @@ func BenchmarkLCOW_Container(b *testing.B) {
  if err := c.Start(ctx); err != nil {
  b.Fatalf("could not start %q: %v", c.ID(), err)
  }
- init := cmd.Create(ctx, b, c, nil, nil)
+ init := testcmd.Create(ctx, b, c, nil, nil)
 
  b.StartTimer()
- cmd.Start(ctx, b, init)
+ if err := init.Start(); err != nil {
+ b.Fatalf("failed to start init command: %v", err)
+ }
  b.StopTimer()
 
- cmd.Kill(ctx, b, init)
- cmd.WaitExitCode(ctx, b, init, 137)
+ testcmd.Kill(ctx, b, init)
+ testcmd.WaitExitCode(ctx, b, init, testcmd.ForcedKilledExitCode)
 
  container.Kill(ctx, b, c)
  container.Wait(ctx, b, c)
@@ -171,10 +176,107 @@ func BenchmarkLCOW_Container(b *testing.B) {
  init := container.Start(ctx, b, c, nil)
 
  b.StartTimer()
- cmd.Kill(ctx, b, init)
- cmd.WaitExitCode(ctx, b, init, 137)
+ if ok, err := init.Process.Kill(ctx); !ok {
+ b.Fatalf("could not deliver kill to init command")
+ } else if err != nil {
+ b.Fatalf("could not kill init command: %v", err)
+ }
+
+ if err := init.Wait(); err != nil {
+ ee := &cmd.ExitError{}
+ if !errors.As(err, &ee) {
+ b.Fatalf("failed to wait on init command: %v", err)
+ }
+ if ee.ExitCode() != testcmd.ForcedKilledExitCode {
+ b.Fatalf("got exit code %d, wanted %d", ee.ExitCode(), testcmd.ForcedKilledExitCode)
+ }
+ }
+ b.StopTimer()
+
+ container.Kill(ctx, b, c)
+ container.Wait(ctx, b, c)
+ cleanup()
+ }
+ })
+
+ b.Run("Exec", func(b *testing.B) {
+ vm := uvm.CreateAndStartLCOWFromOpts(ctx, b, defaultLCOWOptions(b))
+ cache := testlayers.CacheFile(ctx, b, "")
+
+ b.StopTimer()
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ id := cri_util.GenerateID()
+ scratch, _ := testlayers.ScratchSpace(ctx, b, vm, "", "", cache)
+ spec := oci.CreateLinuxSpec(ctx, b, id,
+ oci.DefaultLinuxSpecOpts(id,
+ ctrdoci.WithProcessArgs("/bin/sh", "-c", oci.TailNullArgs),
+ oci.WithWindowsLayerFolders(append(ls, scratch)))...)
+
+ c, _, cleanup := container.Create(ctx, b, vm, spec, id, hcsOwner)
+ init := container.Start(ctx, b, c, nil)
+
+ ps := oci.CreateLinuxSpec(ctx, b, id,
+ oci.DefaultLinuxSpecOpts(id,
+ ctrdoci.WithDefaultPathEnv,
+ ctrdoci.WithProcessArgs("/bin/sh", "-c", oci.TailNullArgs))...,
+ ).Process
+
+ exec := testcmd.Create(ctx, b, c, ps, nil)
+
+ b.StartTimer()
+ if err := exec.Start(); err != nil {
+ b.Fatalf("failed to start %q: %v", strings.Join(exec.Spec.Args, " "), err)
+ }
+ b.StopTimer()
+
+ testcmd.Kill(ctx, b, exec)
+ testcmd.WaitExitCode(ctx, b, exec, testcmd.ForcedKilledExitCode)
+
+ testcmd.Kill(ctx, b, init)
+ testcmd.WaitExitCode(ctx, b, init, testcmd.ForcedKilledExitCode)
+ container.Kill(ctx, b, c)
+ container.Wait(ctx, b, c)
+ cleanup()
+ }
+ })
+
+ b.Run("ExecSync", func(b *testing.B) {
+ vm := uvm.CreateAndStartLCOWFromOpts(ctx, b, defaultLCOWOptions(b))
+ cache := testlayers.CacheFile(ctx, b, "")
+
+ b.StopTimer()
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ id := cri_util.GenerateID()
+ scratch, _ := testlayers.ScratchSpace(ctx, b, vm, "", "", cache)
+ spec := oci.CreateLinuxSpec(ctx, b, id,
+ oci.DefaultLinuxSpecOpts(id,
+ ctrdoci.WithProcessArgs("/bin/sh", "-c", oci.TailNullArgs),
+ oci.WithWindowsLayerFolders(append(ls, scratch)))...)
+
+ c, _, cleanup := container.Create(ctx, b, vm, spec, id, hcsOwner)
+ init := container.Start(ctx, b, c, nil)
+
+ ps := oci.CreateLinuxSpec(ctx, b, id,
+ oci.DefaultLinuxSpecOpts(id,
+ ctrdoci.WithDefaultPathEnv,
+ ctrdoci.WithProcessArgs("/bin/sh", "-c", "true"))...,
+ ).Process
+
+ exec := testcmd.Create(ctx, b, c, ps, nil)
+
+ b.StartTimer()
+ if err := exec.Start(); err != nil {
+ b.Fatalf("failed to start %q: %v", strings.Join(exec.Spec.Args, " "), err)
+ }
+ if err := exec.Wait(); err != nil {
+ b.Fatalf("failed to wait on %q: %v", strings.Join(exec.Spec.Args, " "), err)
+ }
  b.StopTimer()
 
+ testcmd.Kill(ctx, b, init)
+ testcmd.WaitExitCode(ctx, b, init, testcmd.ForcedKilledExitCode)
  container.Kill(ctx, b, c)
  container.Wait(ctx, b, c)
  cleanup()
@@ -206,7 +308,7 @@ func BenchmarkLCOW_Container(b *testing.B) {
  //
  // So ... to test container kill and wait times, we need to first start and wait on the init process
  init := container.Start(ctx, b, c, nil)
- cmd.WaitExitCode(ctx, b, init, 0)
+ testcmd.WaitExitCode(ctx, b, init, 0)
 
  b.StartTimer()
  container.Kill(ctx, b, c)

test/functional/lcow_container_test.go

@@ -44,9 +44,7 @@ func TestLCOW_ContainerLifecycle(t *testing.T) {
  container.Wait(ctx, t, c)
  })
  cmd.Kill(ctx, t, init)
- if e := cmd.Wait(ctx, t, init); e != 137 {
- t.Errorf("got exit code %d, wanted %d", e, 137)
- }
+ cmd.WaitExitCode(ctx, t, init, cmd.ForcedKilledExitCode)
 }
 
 var ioTests = []struct {

test/functional/lcow_policy_test.go

@@ -99,9 +99,7 @@ func Test_GetProperties_WithPolicy(t *testing.T) {
  }
 
  cmd.Kill(ctx, t, init)
- if e := cmd.Wait(ctx, t, init); e != 137 {
- t.Errorf("got exit code %d, wanted %d", e, 137)
- }
+ cmd.WaitExitCode(ctx, t, init, cmd.ForcedKilledExitCode)
  })
  }
 }

test/internal/cmd/cmd.go

@@ -18,6 +18,9 @@ import (
 
 const CopyAfterExitTimeout = time.Second
 
+// ForcedKilledExitCode is the (Linux) exit code when processes are foreably killed.
+const ForcedKilledExitCode = 137
+
 func desc(c *cmd.Cmd) string {
  desc := "init command"
  if c.Spec != nil {