test: run docker tests in parallel

dev-tools/systemtests/container.go

@@ -67,11 +67,6 @@
 	FatalLogMessages []string
 	// MonitorPID will tell tests to specifically check the correctness of process-level monitoring for this PID
 	MonitorPID int
-	// CreateHostProcess: this will start a process with the following args outside of the container,
-	// and use the integration tests to monitor it.
-	// Useful as "monitor random running processes" as a test heuristic tends to be flaky.
-	// This will overrite MonitorPID, so only set either this or MonitorPID.
-	CreateHostProcess *exec.Cmd
 }
 
 // RunResult returns the logs and return code from the container
@@ -131,13 +126,17 @@
 	defer apiClient.Close()
 
 	baseRunner := tr.Runner // some odd recursion happens here if we just refer to tr.Runner
+	parallel := len(cases) > 1
 	for _, tc := range cases {
 		baseRunner.Run(tc.String(), func(t *testing.T) {
-			runner := tr
+			runner := *tr
 			runner.Runner = t
 			runner.CgroupNSMode = tc.nsmode
 			runner.Privileged = tc.priv
 			runner.RunAsUser = tc.user
+			if parallel {
+				t.Parallel()
+			}
 			runner.RunTestsOnDocker(ctx, apiClient)
 		})
 	}
@@ -171,7 +170,9 @@
 	}
 
 	// create monitored process, if we need to
-	tr.createMonitoredProcess(ctx, log)
+	if cmd := tr.createMonitoredProcess(ctx, log); cmd != nil {
+		defer cmd.Cancel()
+	}
 
 	resp := tr.createTestContainer(ctx, log, apiClient)
 
@@ -298,26 +299,27 @@
 	return res
 }
 
-func (tr *DockerTestRunner) createMonitoredProcess(ctx context.Context, logger *logp.Logger) {
-	// if user has specified a process to monitor, start it now
-	// skip if the process has already been created
-	if tr.CreateHostProcess != nil && tr.CreateHostProcess.Process == nil {
-		// We don't need to do this in a channel, but it prevents races between this goroutine
-		// and the rest of test framework
-		startPid := make(chan int)
-		logger.Infof("Creating test Process...")
-		go func() {
-			err := tr.CreateHostProcess.Start()
-			// if the process fails to start up, the resulting tests will fail anyway, so just log it
-			assert.NoError(tr.Runner, err, "error starting monitor process")
-			startPid <- tr.CreateHostProcess.Process.Pid
-
-		}()
-		select {
-		case pid := <-startPid:
-			tr.MonitorPID = pid
-		case <-ctx.Done():
-		}
-		logger.Infof("Monitoring pid %d", tr.MonitorPID)
+func (tr *DockerTestRunner) createMonitoredProcess(ctx context.Context, logger *logp.Logger) *exec.Cmd {
+	if tr.MonitorPID != 0 {
+		return nil
+	}
+	cmd := exec.CommandContext(ctx, "sleep", "240")
+	// We don't need to do this in a channel, but it prevents races between this goroutine
+	// and the rest of test framework
+	startPid := make(chan int)
+	logger.Infof("Creating test Process...")
+	go func() {
+		err := cmd.Start()
+		// if the process fails to start up, the resulting tests will fail anyway, so just log it
+		assert.NoError(tr.Runner, err, "error starting monitor process")
+		startPid <- cmd.Process.Pid
+
+	}()
+	select {
+	case pid := <-startPid:
+		tr.MonitorPID = pid
+	case <-ctx.Done():
 	}
+	logger.Infof("Monitoring pid %d", tr.MonitorPID)
+	return cmd
 }

tests/container_system_mon_test.go

@@ -20,7 +20,6 @@
 import (
 	"context"
 	"os"
-	"os/exec"
 	"path/filepath"
 	"runtime"
 	"strconv"
@@ -42,7 +41,7 @@
 	if runtime.GOOS != "linux" {
 		t.Skip("test is linux-only")
 	}
 	_ = logp.DevelopmentSetup()
 	//manually fetch a kernel process
 	// kernel processes will have a parent pid of 2
 	dir, err := os.Open("/proc")
@@ -96,17 +95,16 @@
 }
 
 func TestProcessMetricsElevatedPerms(t *testing.T) {
 	_ = logp.DevelopmentSetup()
 	ctx, cancel := context.WithTimeout(context.Background(), time.Minute*5)
 	defer cancel()
 	// runs test cases where we do not expect any kind of permissions errors
 	baseRunner := systemtests.DockerTestRunner{
 		Runner:            t,
 		Basepath:          "./metric/system/process",
 		Verbose:           true,
 		Privileged:        true,
 		Testname:          "TestSystemHostFromContainer",
-		CreateHostProcess: exec.Command("sleep", "240"),
 		FatalLogMessages:  []string{"Error fetching PID info for", "Non-fatal error fetching"},
 	}
 
@@ -115,7 +113,7 @@
 }
 
 func TestProcessAllSettings(t *testing.T) {
 	_ = logp.DevelopmentSetup()
 	ctx, cancel := context.WithTimeout(context.Background(), time.Minute*5)
 	defer cancel()
 	// runs test cases where we do not expect any kind of permissions errors
@@ -125,7 +123,6 @@
 		Verbose:           false,
 		Privileged:        true,
 		Testname:          "TestSystemHostFromContainer",
-		CreateHostProcess: exec.Command("sleep", "480"),
 		FatalLogMessages:  []string{"Error fetching PID info for"},
 	}
 
@@ -136,7 +133,7 @@
 }
 
 func TestContainerProcess(t *testing.T) {
 	_ = logp.DevelopmentSetup()
 	ctx, cancel := context.WithTimeout(context.Background(), time.Minute*5)
 	defer cancel()
 	// Make sure that monitoring container procs from within the container still works
@@ -156,7 +153,7 @@
 }
 
 func TestFilesystem(t *testing.T) {
 	_ = logp.DevelopmentSetup()
 	ctx, cancel := context.WithTimeout(context.Background(), time.Minute*5)
 	defer cancel()