rpctest: prevent leaking processes in tests due to panics

This commit adds a `defer` statement at the top of  `TestRpcServer`
which will attempt a `recover` which tears down all active harnesses in
the event that one of the tests causes a panic in the main goroutine.

Before this commit, if a buggy test caused a panic while all integration
tests were being executed, then any active harnesses would fail to be
properly torn down. This would cause the running btcd processes to be
leaked, possibly interfering with future test runs until the process was
manually killed. This commit fixes such behavior.

In order to aide in debugging, when a test panics, the test number is
printed out along with a full stack-trace from the start of the test to
the panic point.

rpcserver_test.go

@@ -8,6 +8,7 @@ import (
 	"bytes"
 	"fmt"
 	"os"
+	"runtime/debug"
 	"testing"
 
 	"github.com/btcsuite/btcd/chaincfg"
@@ -109,25 +110,42 @@ func TestMain(m *testing.M) {
 	// Initialize the primary mining node with a chain of length 125,
 	// providing 25 mature coinbases to allow spending from for testing
 	// purposes.
-	if err = primaryHarness.SetUp(true, 25); err != nil {
+	if err := primaryHarness.SetUp(true, 25); err != nil {
 		fmt.Println("unable to setup test chain: ", err)
 		os.Exit(1)
 	}
 
 	exitCode := m.Run()
 
-	// Clean up the primary harness created above. This includes removing
-	// all temporary directories, and shutting down any created processes.
-	if err := primaryHarness.TearDown(); err != nil {
-		fmt.Println("unable to setup test chain: ", err)
+	// Clean up any active harnesses that are still currently running.This
+	// includes removing all temporary directories, and shutting down any
+	// created processes.
+	if err := rpctest.TearDownAll(); err != nil {
+		fmt.Println("unable to tear down all harnesses: ", err)
 		os.Exit(1)
 	}
 
 	os.Exit(exitCode)
 }
 
 func TestRpcServer(t *testing.T) {
+	var currentTestNum int
+	defer func() {
+		// If one of the integration tests caused a panic within the main
+		// goroutine, then tear down all the harnesses in order to avoid
+		// any leaked btcd processes.
+		if r := recover(); r != nil {
+			fmt.Println("recovering from test panic: ", r)
+			if err := rpctest.TearDownAll(); err != nil {
+				fmt.Println("unable to tear down all harnesses: ", err)
+			}
+			t.Fatalf("test #%v panicked: %s", currentTestNum, debug.Stack())
+		}
+	}()
+
 	for _, testCase := range rpcTestCases {
 		testCase(primaryHarness, t)
+
+		currentTestNum++
 	}
 }