Fix: e2e defunct process (#192)

* TestServer: kill process group

* fix lint

* Makefile test add '-count=1' ignore cache

* Makefile test command add build dependency

* Ctrl-C kill process group

* fix mockStore.GetReceiptsByHash concurrent error

* never release reserved port

* never use the same port

* fix lint

* build tag support unix environment

Makefile

@@ -28,8 +28,8 @@ lint:
 	golangci-lint run -c lint-rule.yaml --timeout=2m
 
 .PHONY: test
-test:
-	PATH=$(shell pwd):${PATH} go test -coverprofile coverage.out -timeout 28m ./...
+test: build
+	PATH=$(shell pwd):${PATH} go test -count=1 -coverprofile coverage.out -timeout 28m ./...
 
 .PHONY: generate-bsd-licenses
 generate-bsd-licenses:

e2e/framework/cmd.go

@@ -0,0 +1,23 @@
+//go:build !aix && !darwin && !dragonfly && !freebsd && !linux && !netbsd && !openbsd && !solaris && !windows && !zos
+// +build !aix,!darwin,!dragonfly,!freebsd,!linux,!netbsd,!openbsd,!solaris,!windows,!zos
+
+package framework
+
+import (
+	"os/exec"
+)
+
+func registerPID(cmd *exec.Cmd) {
+	// ignore
+}
+
+func execCommand(workdir, name string, args ...string) *exec.Cmd {
+	cmd := exec.Command(binaryName, args...)
+	cmd.Dir = workdir
+
+	return cmd
+}
+
+func processKill(cmd *exec.Cmd) error {
+	return cmd.Process.Kill()
+}

e2e/framework/cmd_unix.go

@@ -0,0 +1,58 @@
+//go:build aix || darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris || zos
+// +build aix darwin dragonfly freebsd linux netbsd openbsd solaris zos
+
+package framework
+
+import (
+	"errors"
+	"os"
+	"os/exec"
+	"os/signal"
+	"syscall"
+)
+
+var pids = []int{}
+
+func init() {
+	go func() {
+		// wait os.Interrupt signal
+		c := make(chan os.Signal, 1)
+		signal.Notify(c, os.Interrupt)
+
+		for sig := range c {
+			if sig == os.Interrupt {
+				// kill the whole process group
+				for _, pid := range pids {
+					_ = syscall.Kill(-pid, syscall.SIGKILL)
+				}
+			}
+
+			os.Exit(1)
+		}
+	}()
+}
+
+func registerPID(cmd *exec.Cmd) {
+	pids = append(pids, cmd.Process.Pid)
+}
+
+func execCommand(workdir, name string, args ...string) *exec.Cmd {
+	cmd := exec.Command(binaryName, args...)
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		Setpgid: true,
+		Pgid:    0,
+	}
+	cmd.Dir = workdir
+
+	return cmd
+}
+
+func processKill(cmd *exec.Cmd) error {
+	// kill the whole process group
+	err := syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)
+	if errors.Is(syscall.ESRCH, err) {
+		return nil
+	}
+
+	return err
+}

e2e/framework/cmd_windows.go

@@ -0,0 +1,23 @@
+//go:build windows
+// +build windows
+
+package framework
+
+import (
+	"os/exec"
+)
+
+func registerPID(cmd *exec.Cmd) {
+	// ignore
+}
+
+func execCommand(workdir, name string, args ...string) *exec.Cmd {
+	cmd := exec.Command(binaryName, args...)
+	cmd.Dir = workdir
+
+	return cmd
+}
+
+func processKill(cmd *exec.Cmd) error {
+	return cmd.Process.Kill()
+}

e2e/framework/helper.go

@@ -381,18 +381,15 @@ func FindAvailablePort(from, to int) *ReservedPort {
 }
 
 func FindAvailablePorts(n, from, to int) ([]ReservedPort, error) {
-	ports := make([]ReservedPort, 0, n)
+	ports := []ReservedPort{}
 	nextFrom := from
 
 	for i := 0; i < n; i++ {
 		newPort := FindAvailablePort(nextFrom, to)
 		if newPort == nil {
-			// Close current reserved ports
-			for _, p := range ports {
-				p.Close()
-			}
+			nextFrom++
 
-			return nil, errors.New("couldn't reserve required number of ports")
+			continue
 		}
 
 		ports = append(ports, *newPort)

e2e/framework/testserver.go

@@ -13,6 +13,7 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+	"sync"
 	"testing"
 	"time"
 
@@ -44,11 +45,13 @@ import (
 type TestServerConfigCallback func(*TestServerConfig)
 
 const (
-	serverIP    = "127.0.0.1"
-	initialPort = 12000
-	binaryName  = "dogechain"
+	serverIP   = "127.0.0.1"
+	binaryName = "dogechain"
 )
 
+var lock sync.Mutex
+var initialPort = 12000
+
 type TestServer struct {
 	t *testing.T
 
@@ -59,12 +62,17 @@ type TestServer struct {
 func NewTestServer(t *testing.T, rootDir string, callback TestServerConfigCallback) *TestServer {
 	t.Helper()
 
-	// Reserve ports
-	ports, err := FindAvailablePorts(3, initialPort, initialPort+10000)
+	lock.Lock()
+	defer lock.Unlock()
+
+	// never use the same port
+	ports, err := FindAvailablePorts(3, initialPort, 65534)
 	if err != nil {
 		t.Fatal(err)
 	}
 
+	initialPort = ports[2].Port() + 1
+
 	// Sets the services to start on open ports
 	config := &TestServerConfig{
 		ReservedPorts: ports,
@@ -170,7 +178,7 @@ func (t *TestServer) Stop() {
 	t.ReleaseReservedPorts()
 
 	if t.cmd != nil {
-		if err := t.cmd.Process.Kill(); err != nil {
+		if err := processKill(t.cmd); err != nil {
 			t.t.Error(err)
 		}
 	}
@@ -194,9 +202,7 @@ func (t *TestServer) SecretsInit() (*InitIBFTResult, error) {
 	args = append(args, commandSlice...)
 	args = append(args, "--data-dir", t.Config.IBFTDir)
 
-	cmd := exec.Command(binaryName, args...)
-	cmd.Dir = t.Config.RootDir
-
+	cmd := execCommand(t.Config.RootDir, binaryName, args...)
 	if _, err := cmd.Output(); err != nil {
 		return nil, err
 	}
@@ -312,8 +318,7 @@ func (t *TestServer) GenerateGenesis() error {
 		args = append(args, "--bridge-signer", signer.String())
 	}
 
-	cmd := exec.Command(binaryName, args...)
-	cmd.Dir = t.Config.RootDir
+	cmd := execCommand(t.Config.RootDir, binaryName, args...)
 
 	return cmd.Run()
 }
@@ -374,8 +379,7 @@ func (t *TestServer) Start(ctx context.Context) error {
 	t.ReleaseReservedPorts()
 
 	// Start the server
-	t.cmd = exec.Command(binaryName, args...)
-	t.cmd.Dir = t.Config.RootDir
+	t.cmd = execCommand(t.Config.RootDir, binaryName, args...)
 
 	if t.Config.ShowsLog {
 		stdout := io.Writer(os.Stdout)
@@ -387,6 +391,13 @@ func (t *TestServer) Start(ctx context.Context) error {
 		return err
 	}
 
+	registerPID(t.cmd)
+
+	// fix defunct process
+	go func() {
+		_ = t.cmd.Wait()
+	}()
+
 	_, err := tests.RetryUntilTimeout(ctx, func() (interface{}, bool) {
 		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 		defer cancel()
@@ -426,8 +437,7 @@ func (t *TestServer) SwitchIBFTType(typ ibft.MechanismType, from uint64, to, dep
 	}
 
 	// Start the server
-	t.cmd = exec.Command(binaryName, args...)
-	t.cmd.Dir = t.Config.RootDir
+	t.cmd = execCommand(t.Config.RootDir, binaryName, args...)
 
 	if t.Config.ShowsLog {
 		stdout := io.Writer(os.Stdout)

jsonrpc/mocks_test.go

@@ -72,6 +72,9 @@ func (m *mockStore) emitEvent(evnt *mockEvent) {
 		OldChain: []*types.Header{},
 	}
 
+	m.receiptsLock.Lock()
+	defer m.receiptsLock.Unlock()
+
 	for _, i := range evnt.NewChain {
 		m.receipts[i.header.Hash] = i.receipts
 		bEvnt.NewChain = append(bEvnt.NewChain, i.header)

protocol/testing.go

@@ -51,7 +51,10 @@ func CreateSyncer(t *testing.T, blockchain blockchainShim, serverCfg *func(c *ne
 	}
 
 	srv, createErr := network.CreateServer(&network.CreateServerParams{
-		ConfigCallback: *serverCfg,
+		ConfigCallback: func(c *network.Config) {
+			c.DataDir = t.TempDir()
+			(*serverCfg)(c)
+		},
 	})
 	if createErr != nil {
 		t.Fatalf("Unable to create networking server, %v", createErr)