From 028bc7e1b4104db1937f0966a946557659c53b30 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?William=20Tis=C3=A4ter?= <william@defunct.cc>
Date: Wed, 7 May 2014 02:48:25 +0200
Subject: [PATCH 01/39] Proof of concept using `WaitForResult` in tests

---
 consul/catalog_endpoint_test.go | 11 ++++++-----
 testutil/wait.go                | 26 ++++++++++++++++++++++++++
 2 files changed, 32 insertions(+), 5 deletions(-)
 create mode 100644 testutil/wait.go

diff --git a/consul/catalog_endpoint_test.go b/consul/catalog_endpoint_test.go
index 3a174ce45637..a553538b8946 100644
--- a/consul/catalog_endpoint_test.go
+++ b/consul/catalog_endpoint_test.go
@@ -2,6 +2,7 @@ package consul
 
 import (
 	"fmt"
+	"github.com/hashicorp/consul/testutil"
 	"github.com/hashicorp/consul/consul/structs"
 	"net/rpc"
 	"os"
@@ -35,12 +36,12 @@ func TestCatalogRegister(t *testing.T) {
 		t.Fatalf("err: %v", err)
 	}
 
-	// Wait for leader
-	time.Sleep(100 * time.Millisecond)
-
-	if err := client.Call("Catalog.Register", &arg, &out); err != nil {
+	testutil.WaitForResult(func() (bool, error) {
+		err := client.Call("Catalog.Register", &arg, &out)
+		return err == nil, err
+	}, func(err error) {
 		t.Fatalf("err: %v", err)
-	}
+	})
 }
 
 func TestCatalogRegister_ForwardLeader(t *testing.T) {
diff --git a/testutil/wait.go b/testutil/wait.go
new file mode 100644
index 000000000000..6071c94ceaad
--- /dev/null
+++ b/testutil/wait.go
@@ -0,0 +1,26 @@
+package testutil
+
+import (
+	"time"
+)
+
+type testFn func() (bool, error)
+type errorFn func(error)
+
+func WaitForResult(test testFn, error errorFn) {
+	retries := 500  // 5 seconds timeout
+
+	for retries > 0 {
+		time.Sleep(10 * time.Millisecond)
+		retries--
+
+		success, err := test()
+		if success {
+			return
+		}
+
+		if retries == 0 {
+			error(err)
+		}
+	}
+}

From f1b9dbe5c3ffa0cebd9f0c287e4cd08e8fc9dee0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?William=20Tis=C3=A4ter?= <william@defunct.cc>
Date: Wed, 7 May 2014 11:43:42 +0200
Subject: [PATCH 02/39] Add `testutils.WaitForLeader` and truly wait

---
 command/agent/dns_test.go | 68 +++++++++++++++++++--------------------
 testutil/wait.go          | 18 +++++++++--
 2 files changed, 50 insertions(+), 36 deletions(-)

diff --git a/command/agent/dns_test.go b/command/agent/dns_test.go
index 52f9eae7384e..e874e3f649df 100644
--- a/command/agent/dns_test.go
+++ b/command/agent/dns_test.go
@@ -2,12 +2,12 @@ package agent
 
 import (
 	"fmt"
+	"github.com/hashicorp/consul/testutil"
 	"github.com/hashicorp/consul/consul/structs"
 	"github.com/miekg/dns"
 	"os"
 	"strings"
 	"testing"
-	"time"
 )
 
 func makeDNSServer(t *testing.T) (string, *DNSServer) {
@@ -65,15 +65,15 @@ func TestDNS_NodeLookup(t *testing.T) {
 	defer os.RemoveAll(dir)
 	defer srv.agent.Shutdown()
 
-	// Wait for leader
-	time.Sleep(100 * time.Millisecond)
-
 	// Register node
 	args := &structs.RegisterRequest{
 		Datacenter: "dc1",
 		Node:       "foo",
 		Address:    "127.0.0.1",
 	}
+
+	testutil.WaitForLeader(t, srv.agent.RPC, args)
+
 	var out struct{}
 	if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
 		t.Fatalf("err: %v", err)
@@ -129,15 +129,15 @@ func TestDNS_NodeLookup_PeriodName(t *testing.T) {
 	defer os.RemoveAll(dir)
 	defer srv.agent.Shutdown()
 
-	// Wait for leader
-	time.Sleep(100 * time.Millisecond)
-
 	// Register node with period in name
 	args := &structs.RegisterRequest{
 		Datacenter: "dc1",
 		Node:       "foo.bar",
 		Address:    "127.0.0.1",
 	}
+
+	testutil.WaitForLeader(t, srv.agent.RPC, args)
+
 	var out struct{}
 	if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
 		t.Fatalf("err: %v", err)
@@ -171,15 +171,15 @@ func TestDNS_NodeLookup_AAAA(t *testing.T) {
 	defer os.RemoveAll(dir)
 	defer srv.agent.Shutdown()
 
-	// Wait for leader
-	time.Sleep(100 * time.Millisecond)
-
 	// Register node
 	args := &structs.RegisterRequest{
 		Datacenter: "dc1",
 		Node:       "bar",
 		Address:    "::4242:4242",
 	}
+
+	testutil.WaitForLeader(t, srv.agent.RPC, args)
+
 	var out struct{}
 	if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
 		t.Fatalf("err: %v", err)
@@ -213,15 +213,15 @@ func TestDNS_NodeLookup_CNAME(t *testing.T) {
 	defer os.RemoveAll(dir)
 	defer srv.agent.Shutdown()
 
-	// Wait for leader
-	time.Sleep(100 * time.Millisecond)
-
 	// Register node
 	args := &structs.RegisterRequest{
 		Datacenter: "dc1",
 		Node:       "google",
 		Address:    "www.google.com",
 	}
+
+	testutil.WaitForLeader(t, srv.agent.RPC, args)
+
 	var out struct{}
 	if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
 		t.Fatalf("err: %v", err)
@@ -256,9 +256,6 @@ func TestDNS_ServiceLookup(t *testing.T) {
 	defer os.RemoveAll(dir)
 	defer srv.agent.Shutdown()
 
-	// Wait for leader
-	time.Sleep(100 * time.Millisecond)
-
 	// Register node
 	args := &structs.RegisterRequest{
 		Datacenter: "dc1",
@@ -270,6 +267,9 @@ func TestDNS_ServiceLookup(t *testing.T) {
 			Port:    12345,
 		},
 	}
+
+	testutil.WaitForLeader(t, srv.agent.RPC, args)
+
 	var out struct{}
 	if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
 		t.Fatalf("err: %v", err)
@@ -317,9 +317,6 @@ func TestDNS_ServiceLookup_TagPeriod(t *testing.T) {
 	defer os.RemoveAll(dir)
 	defer srv.agent.Shutdown()
 
-	// Wait for leader
-	time.Sleep(100 * time.Millisecond)
-
 	// Register node
 	args := &structs.RegisterRequest{
 		Datacenter: "dc1",
@@ -331,6 +328,9 @@ func TestDNS_ServiceLookup_TagPeriod(t *testing.T) {
 			Port:    12345,
 		},
 	}
+
+	testutil.WaitForLeader(t, srv.agent.RPC, args)
+
 	var out struct{}
 	if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
 		t.Fatalf("err: %v", err)
@@ -378,9 +378,6 @@ func TestDNS_ServiceLookup_Dedup(t *testing.T) {
 	defer os.RemoveAll(dir)
 	defer srv.agent.Shutdown()
 
-	// Wait for leader
-	time.Sleep(100 * time.Millisecond)
-
 	// Register node
 	args := &structs.RegisterRequest{
 		Datacenter: "dc1",
@@ -392,6 +389,9 @@ func TestDNS_ServiceLookup_Dedup(t *testing.T) {
 			Port:    12345,
 		},
 	}
+
+	testutil.WaitForLeader(t, srv.agent.RPC, args)
+
 	var out struct{}
 	if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
 		t.Fatalf("err: %v", err)
@@ -455,9 +455,6 @@ func TestDNS_ServiceLookup_Dedup_SRV(t *testing.T) {
 	defer os.RemoveAll(dir)
 	defer srv.agent.Shutdown()
 
-	// Wait for leader
-	time.Sleep(100 * time.Millisecond)
-
 	// Register node
 	args := &structs.RegisterRequest{
 		Datacenter: "dc1",
@@ -469,6 +466,9 @@ func TestDNS_ServiceLookup_Dedup_SRV(t *testing.T) {
 			Port:    12345,
 		},
 	}
+
+	testutil.WaitForLeader(t, srv.agent.RPC, args)
+
 	var out struct{}
 	if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
 		t.Fatalf("err: %v", err)
@@ -584,9 +584,6 @@ func TestDNS_ServiceLookup_FilterCritical(t *testing.T) {
 	defer os.RemoveAll(dir)
 	defer srv.agent.Shutdown()
 
-	// Wait for leader
-	time.Sleep(100 * time.Millisecond)
-
 	// Register nodes
 	args := &structs.RegisterRequest{
 		Datacenter: "dc1",
@@ -603,6 +600,9 @@ func TestDNS_ServiceLookup_FilterCritical(t *testing.T) {
 			Status:  structs.HealthCritical,
 		},
 	}
+
+	testutil.WaitForLeader(t, srv.agent.RPC, args)
+
 	var out struct{}
 	if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
 		t.Fatalf("err: %v", err)
@@ -648,9 +648,6 @@ func TestDNS_ServiceLookup_Randomize(t *testing.T) {
 	defer os.RemoveAll(dir)
 	defer srv.agent.Shutdown()
 
-	// Wait for leader
-	time.Sleep(100 * time.Millisecond)
-
 	// Register nodes
 	for i := 0; i < 3*maxServiceResponses; i++ {
 		args := &structs.RegisterRequest{
@@ -662,6 +659,9 @@ func TestDNS_ServiceLookup_Randomize(t *testing.T) {
 				Port:    8000,
 			},
 		}
+
+		testutil.WaitForLeader(t, srv.agent.RPC, args)
+
 		var out struct{}
 		if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
 			t.Fatalf("err: %v", err)
@@ -712,9 +712,6 @@ func TestDNS_ServiceLookup_CNAME(t *testing.T) {
 	defer os.RemoveAll(dir)
 	defer srv.agent.Shutdown()
 
-	// Wait for leader
-	time.Sleep(100 * time.Millisecond)
-
 	// Register node
 	args := &structs.RegisterRequest{
 		Datacenter: "dc1",
@@ -725,6 +722,9 @@ func TestDNS_ServiceLookup_CNAME(t *testing.T) {
 			Port:    80,
 		},
 	}
+
+	testutil.WaitForLeader(t, srv.agent.RPC, args)
+
 	var out struct{}
 	if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
 		t.Fatalf("err: %v", err)
diff --git a/testutil/wait.go b/testutil/wait.go
index 6071c94ceaad..6cc2d95597ff 100644
--- a/testutil/wait.go
+++ b/testutil/wait.go
@@ -2,16 +2,18 @@ package testutil
 
 import (
 	"time"
+	"testing"
+	"github.com/hashicorp/consul/consul/structs"
 )
 
 type testFn func() (bool, error)
 type errorFn func(error)
 
 func WaitForResult(test testFn, error errorFn) {
-	retries := 500  // 5 seconds timeout
+	retries := 100  // 5 seconds timeout
 
 	for retries > 0 {
-		time.Sleep(10 * time.Millisecond)
+		time.Sleep(50 * time.Millisecond)
 		retries--
 
 		success, err := test()
@@ -24,3 +26,15 @@ func WaitForResult(test testFn, error errorFn) {
 		}
 	}
 }
+
+type clientRPC func(string, interface {}, interface {}) error
+
+func WaitForLeader(t *testing.T, rpc clientRPC, args interface{}) {
+	WaitForResult(func() (bool, error) {
+		var out structs.IndexedNodes
+		err := rpc("Catalog.ListNodes", args, &out)
+		return out.QueryMeta.KnownLeader, err
+	}, func(err error) {
+		t.Fatalf("failed to find leader: %v", err)
+	})
+}

From aa00bfff7d49a7cc4a5b149c0b14a3e63a44c44d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?William=20Tisa=CC=88ter?= <william@defunct.cc>
Date: Wed, 7 May 2014 13:53:29 +0200
Subject: [PATCH 03/39] Fix `TestCatalogListNodes_StaleRaad` with
 `WaitForLeader`

---
 consul/catalog_endpoint_test.go | 15 ++++++++-------
 testutil/wait.go                |  7 ++++---
 2 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/consul/catalog_endpoint_test.go b/consul/catalog_endpoint_test.go
index a553538b8946..3f01b49db66d 100644
--- a/consul/catalog_endpoint_test.go
+++ b/consul/catalog_endpoint_test.go
@@ -2,8 +2,8 @@ package consul
 
 import (
 	"fmt"
-	"github.com/hashicorp/consul/testutil"
 	"github.com/hashicorp/consul/consul/structs"
+	"github.com/hashicorp/consul/testutil"
 	"net/rpc"
 	"os"
 	"sort"
@@ -254,8 +254,13 @@ func TestCatalogListNodes_StaleRaad(t *testing.T) {
 		t.Fatalf("err: %v", err)
 	}
 
-	// Wait for a leader
-	time.Sleep(100 * time.Millisecond)
+	args := structs.DCSpecificRequest{
+		Datacenter:   "dc1",
+		QueryOptions: structs.QueryOptions{AllowStale: true},
+	}
+
+	testutil.WaitForLeader(t, client1.Call, args)
+	testutil.WaitForLeader(t, client2.Call, args)
 
 	// Use the follower as the client
 	var client *rpc.Client
@@ -271,10 +276,6 @@ func TestCatalogListNodes_StaleRaad(t *testing.T) {
 		s2.fsm.State().EnsureNode(1, structs.Node{"foo", "127.0.0.1"})
 	}
 
-	args := structs.DCSpecificRequest{
-		Datacenter:   "dc1",
-		QueryOptions: structs.QueryOptions{AllowStale: true},
-	}
 	var out structs.IndexedNodes
 	if err := client.Call("Catalog.ListNodes", &args, &out); err != nil {
 		t.Fatalf("err: %v", err)
diff --git a/testutil/wait.go b/testutil/wait.go
index 6cc2d95597ff..78ef2d140cc3 100644
--- a/testutil/wait.go
+++ b/testutil/wait.go
@@ -27,14 +27,15 @@ func WaitForResult(test testFn, error errorFn) {
 	}
 }
 
-type clientRPC func(string, interface {}, interface {}) error
+type rpcFn func(string, interface {}, interface {}) error
 
-func WaitForLeader(t *testing.T, rpc clientRPC, args interface{}) {
+func WaitForLeader(t *testing.T, rpc rpcFn, args interface{}) structs.IndexedNodes {
+	var out structs.IndexedNodes
 	WaitForResult(func() (bool, error) {
-		var out structs.IndexedNodes
 		err := rpc("Catalog.ListNodes", args, &out)
 		return out.QueryMeta.KnownLeader, err
 	}, func(err error) {
 		t.Fatalf("failed to find leader: %v", err)
 	})
+	return out
 }

From dde3b90632f83ed843281cbe9acd406f33a5b923 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?William=20Tis=C3=A4ter?= <william@defunct.cc>
Date: Wed, 7 May 2014 22:45:16 +0200
Subject: [PATCH 04/39] Remove all sleeps from `force_leave_test.go`

---
 command/force_leave_test.go | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/command/force_leave_test.go b/command/force_leave_test.go
index 6b66b4fb1c05..40a2224f7b82 100644
--- a/command/force_leave_test.go
+++ b/command/force_leave_test.go
@@ -2,12 +2,12 @@ package command
 
 import (
 	"fmt"
+	"github.com/hashicorp/consul/testutil"
 	"github.com/hashicorp/serf/serf"
-	"github.com/hashicorp/serf/testutil"
 	"github.com/mitchellh/cli"
 	"strings"
 	"testing"
-	"time"
+	"errors"
 )
 
 func TestForceLeaveCommand_implements(t *testing.T) {
@@ -26,13 +26,9 @@ func TestForceLeaveCommandRun(t *testing.T) {
 		t.Fatalf("err: %s", err)
 	}
 
-	testutil.Yield()
-
 	// Forcibly shutdown a2 so that it appears "failed" in a1
 	a2.Shutdown()
 
-	time.Sleep(time.Second)
-
 	ui := new(cli.MockUi)
 	c := &ForceLeaveCommand{Ui: ui}
 	args := []string{
@@ -50,9 +46,14 @@ func TestForceLeaveCommandRun(t *testing.T) {
 		t.Fatalf("should have 2 members: %#v", m)
 	}
 
-	if m[1].Status != serf.StatusLeft {
-		t.Fatalf("should be left: %#v", m[1])
-	}
+	testutil.WaitForResult(func() (bool, error) {
+		m = a1.agent.LANMembers()
+		success := m[1].Status == serf.StatusLeft
+		err := errors.New(m[1].Status.String())
+		return success, err
+	}, func(err error) {
+		t.Fatalf("member status is %v, not left", err)
+	})
 }
 
 func TestForceLeaveCommandRun_noAddrs(t *testing.T) {

From ab209c0176f2caa80cf431ab9a69189c53ac9297 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?William=20Tis=C3=A4ter?= <william@defunct.cc>
Date: Wed, 7 May 2014 22:50:18 +0200
Subject: [PATCH 05/39] Remove all sleeps from `agent_endpoint_test.go`

---
 command/agent/agent_endpoint_test.go | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/command/agent/agent_endpoint_test.go b/command/agent/agent_endpoint_test.go
index 15182e495b53..78406de731a5 100644
--- a/command/agent/agent_endpoint_test.go
+++ b/command/agent/agent_endpoint_test.go
@@ -2,12 +2,14 @@ package agent
 
 import (
 	"fmt"
+	"github.com/hashicorp/consul/testutil"
 	"github.com/hashicorp/consul/consul/structs"
 	"github.com/hashicorp/serf/serf"
 	"net/http"
 	"os"
 	"testing"
 	"time"
+	"errors"
 )
 
 func TestHTTPAgentServices(t *testing.T) {
@@ -189,9 +191,7 @@ func TestHTTPAgentForceLeave(t *testing.T) {
 		t.Fatalf("err: %v", err)
 	}
 
-	// Shutdown, wait for detection
 	a2.Shutdown()
-	time.Sleep(500 * time.Millisecond)
 
 	// Force leave now
 	req, err := http.NewRequest("GET", fmt.Sprintf("/v1/agent/force-leave/%s", a2.config.NodeName), nil)
@@ -207,11 +207,13 @@ func TestHTTPAgentForceLeave(t *testing.T) {
 		t.Fatalf("Err: %v", obj)
 	}
 
-	// SHould be left
-	mem := srv.agent.LANMembers()
-	if mem[1].Status != serf.StatusLeft {
-		t.Fatalf("should have left: %v", mem)
-	}
+	testutil.WaitForResult(func() (bool, error) {
+		m := srv.agent.LANMembers()
+		success := m[1].Status == serf.StatusLeft
+		return success, errors.New(m[1].Status.String())
+	}, func(err error) {
+		t.Fatalf("member status is %v, should be left", err)
+	})
 }
 
 func TestHTTPAgentRegisterCheck(t *testing.T) {

From 9027d63059d8103a3eb2e139143f8ab7a691b2ec Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?William=20Tis=C3=A4ter?= <william@defunct.cc>
Date: Wed, 7 May 2014 22:50:34 +0200
Subject: [PATCH 06/39] Fix formatting in `TestForceLeaveCommandRun`

---
 command/force_leave_test.go | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/command/force_leave_test.go b/command/force_leave_test.go
index 40a2224f7b82..8189d41d2aec 100644
--- a/command/force_leave_test.go
+++ b/command/force_leave_test.go
@@ -49,10 +49,9 @@ func TestForceLeaveCommandRun(t *testing.T) {
 	testutil.WaitForResult(func() (bool, error) {
 		m = a1.agent.LANMembers()
 		success := m[1].Status == serf.StatusLeft
-		err := errors.New(m[1].Status.String())
-		return success, err
+		return success, errors.New(m[1].Status.String())
 	}, func(err error) {
-		t.Fatalf("member status is %v, not left", err)
+		t.Fatalf("member status is %v, should be left", err)
 	})
 }
 

From 1a32697b2bb676eee57cce94f0d95ac2a2261393 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?William=20Tis=C3=A4ter?= <william@defunct.cc>
Date: Wed, 7 May 2014 23:02:59 +0200
Subject: [PATCH 07/39] Remove all sleeps from `catalog_endpoint_test.go`

---
 command/agent/catalog_endpoint_test.go | 72 +++++++++++++-------------
 1 file changed, 36 insertions(+), 36 deletions(-)

diff --git a/command/agent/catalog_endpoint_test.go b/command/agent/catalog_endpoint_test.go
index 2d6dc39d4ea5..fa553d112a34 100644
--- a/command/agent/catalog_endpoint_test.go
+++ b/command/agent/catalog_endpoint_test.go
@@ -2,6 +2,7 @@ package agent
 
 import (
 	"fmt"
+	"github.com/hashicorp/consul/testutil"
 	"github.com/hashicorp/consul/consul/structs"
 	"net/http"
 	"net/http/httptest"
@@ -16,18 +17,19 @@ func TestCatalogRegister(t *testing.T) {
 	defer srv.Shutdown()
 	defer srv.agent.Shutdown()
 
-	// Wait for a leader
-	time.Sleep(100 * time.Millisecond)
+	args := &structs.RegisterRequest{
+		Datacenter: "dc1",
+		Node:       "foo",
+		Address:    "127.0.0.1",
+	}
+
+	testutil.WaitForLeader(t, srv.agent.RPC, args)
 
 	// Register node
 	req, err := http.NewRequest("GET", "/v1/catalog/register", nil)
 	if err != nil {
 		t.Fatalf("err: %v", err)
 	}
-	args := &structs.RegisterRequest{
-		Node:    "foo",
-		Address: "127.0.0.1",
-	}
 	req.Body = encodeReq(args)
 
 	obj, err := srv.CatalogRegister(nil, req)
@@ -47,17 +49,18 @@ func TestCatalogDeregister(t *testing.T) {
 	defer srv.Shutdown()
 	defer srv.agent.Shutdown()
 
-	// Wait for a leader
-	time.Sleep(100 * time.Millisecond)
+	args := &structs.DeregisterRequest{
+		Datacenter: "dc1",
+		Node:       "foo",
+	}
+
+	testutil.WaitForLeader(t, srv.agent.RPC, args)
 
 	// Register node
 	req, err := http.NewRequest("GET", "/v1/catalog/deregister", nil)
 	if err != nil {
 		t.Fatalf("err: %v", err)
 	}
-	args := &structs.DeregisterRequest{
-		Node: "foo",
-	}
 	req.Body = encodeReq(args)
 
 	obj, err := srv.CatalogDeregister(nil, req)
@@ -77,9 +80,6 @@ func TestCatalogDatacenters(t *testing.T) {
 	defer srv.Shutdown()
 	defer srv.agent.Shutdown()
 
-	// Wait for initialization
-	time.Sleep(10 * time.Millisecond)
-
 	obj, err := srv.CatalogDatacenters(nil, nil)
 	if err != nil {
 		t.Fatalf("err: %v", err)
@@ -97,15 +97,15 @@ func TestCatalogNodes(t *testing.T) {
 	defer srv.Shutdown()
 	defer srv.agent.Shutdown()
 
-	// Wait for a leader
-	time.Sleep(100 * time.Millisecond)
-
-	// Register node
 	args := &structs.RegisterRequest{
 		Datacenter: "dc1",
 		Node:       "foo",
 		Address:    "127.0.0.1",
 	}
+
+	testutil.WaitForLeader(t, srv.agent.RPC, args)
+
+	// Register node
 	var out struct{}
 	if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
 		t.Fatalf("err: %v", err)
@@ -137,13 +137,13 @@ func TestCatalogNodes_Blocking(t *testing.T) {
 	defer srv.Shutdown()
 	defer srv.agent.Shutdown()
 
-	// Wait for a leader
-	time.Sleep(100 * time.Millisecond)
-
-	// Register node
 	args := &structs.DCSpecificRequest{
 		Datacenter: "dc1",
 	}
+
+	testutil.WaitForLeader(t, srv.agent.RPC, args)
+
+	// Register node
 	var out structs.IndexedNodes
 	if err := srv.agent.RPC("Catalog.ListNodes", *args, &out); err != nil {
 		t.Fatalf("err: %v", err)
@@ -152,7 +152,7 @@ func TestCatalogNodes_Blocking(t *testing.T) {
 	// Do an update in a little while
 	start := time.Now()
 	go func() {
-		time.Sleep(100 * time.Millisecond)
+		time.Sleep(50 * time.Millisecond)
 		args := &structs.RegisterRequest{
 			Datacenter: "dc1",
 			Node:       "foo",
@@ -178,7 +178,7 @@ func TestCatalogNodes_Blocking(t *testing.T) {
 	}
 
 	// Should block for a while
-	if time.Now().Sub(start) < 100*time.Millisecond {
+	if time.Now().Sub(start) < 50 * time.Millisecond {
 		t.Fatalf("too fast")
 	}
 
@@ -198,10 +198,6 @@ func TestCatalogServices(t *testing.T) {
 	defer srv.Shutdown()
 	defer srv.agent.Shutdown()
 
-	// Wait for a leader
-	time.Sleep(100 * time.Millisecond)
-
-	// Register node
 	args := &structs.RegisterRequest{
 		Datacenter: "dc1",
 		Node:       "foo",
@@ -210,6 +206,10 @@ func TestCatalogServices(t *testing.T) {
 			Service: "api",
 		},
 	}
+
+	testutil.WaitForLeader(t, srv.agent.RPC, args)
+
+	// Register node
 	var out struct{}
 	if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
 		t.Fatalf("err: %v", err)
@@ -240,10 +240,6 @@ func TestCatalogServiceNodes(t *testing.T) {
 	defer srv.Shutdown()
 	defer srv.agent.Shutdown()
 
-	// Wait for a leader
-	time.Sleep(100 * time.Millisecond)
-
-	// Register node
 	args := &structs.RegisterRequest{
 		Datacenter: "dc1",
 		Node:       "foo",
@@ -253,6 +249,10 @@ func TestCatalogServiceNodes(t *testing.T) {
 			Tags:    []string{"a"},
 		},
 	}
+
+	testutil.WaitForLeader(t, srv.agent.RPC, args)
+
+	// Register node
 	var out struct{}
 	if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
 		t.Fatalf("err: %v", err)
@@ -283,10 +283,6 @@ func TestCatalogNodeServices(t *testing.T) {
 	defer srv.Shutdown()
 	defer srv.agent.Shutdown()
 
-	// Wait for a leader
-	time.Sleep(100 * time.Millisecond)
-
-	// Register node
 	args := &structs.RegisterRequest{
 		Datacenter: "dc1",
 		Node:       "foo",
@@ -296,6 +292,10 @@ func TestCatalogNodeServices(t *testing.T) {
 			Tags:    []string{"a"},
 		},
 	}
+
+	testutil.WaitForLeader(t, srv.agent.RPC, args)
+
+	// Register node
 	var out struct{}
 	if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
 		t.Fatalf("err: %v", err)

From e4598284fdc56749a593e6df33edcde14ccbc861 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?William=20Tis=C3=A4ter?= <william@defunct.cc>
Date: Wed, 7 May 2014 23:12:53 +0200
Subject: [PATCH 08/39] Remove all sleeps from `health_endpoint_test.go`

---
 command/agent/health_endpoint_test.go | 39 ++++++++++++++++-----------
 1 file changed, 24 insertions(+), 15 deletions(-)

diff --git a/command/agent/health_endpoint_test.go b/command/agent/health_endpoint_test.go
index e51f26084079..2f669846ca11 100644
--- a/command/agent/health_endpoint_test.go
+++ b/command/agent/health_endpoint_test.go
@@ -2,12 +2,12 @@ package agent
 
 import (
 	"fmt"
+	"github.com/hashicorp/consul/testutil"
 	"github.com/hashicorp/consul/consul/structs"
 	"net/http"
 	"net/http/httptest"
 	"os"
 	"testing"
-	"time"
 )
 
 func TestHealthChecksInState(t *testing.T) {
@@ -16,8 +16,11 @@ func TestHealthChecksInState(t *testing.T) {
 	defer srv.Shutdown()
 	defer srv.agent.Shutdown()
 
-	// Wait for a leader
-	time.Sleep(100 * time.Millisecond)
+	args := &structs.RegisterRequest{
+		Datacenter: "dc1",
+	}
+
+	testutil.WaitForLeader(t, srv.agent.RPC, args)
 
 	req, err := http.NewRequest("GET", "/v1/health/state/passing?dc=dc1", nil)
 	if err != nil {
@@ -44,8 +47,11 @@ func TestHealthNodeChecks(t *testing.T) {
 	defer srv.Shutdown()
 	defer srv.agent.Shutdown()
 
-	// Wait for a leader
-	time.Sleep(100 * time.Millisecond)
+	args := &structs.RegisterRequest{
+		Datacenter: "dc1",
+	}
+
+	testutil.WaitForLeader(t, srv.agent.RPC, args)
 
 	req, err := http.NewRequest("GET",
 		fmt.Sprintf("/v1/health/node/%s?dc=dc1", srv.agent.config.NodeName), nil)
@@ -73,10 +79,6 @@ func TestHealthServiceChecks(t *testing.T) {
 	defer srv.Shutdown()
 	defer srv.agent.Shutdown()
 
-	// Wait for a leader
-	time.Sleep(100 * time.Millisecond)
-
-	// Create a service check
 	args := &structs.RegisterRequest{
 		Datacenter: "dc1",
 		Node:       srv.agent.config.NodeName,
@@ -87,6 +89,10 @@ func TestHealthServiceChecks(t *testing.T) {
 			ServiceID: "consul",
 		},
 	}
+
+	testutil.WaitForLeader(t, srv.agent.RPC, args)
+
+	// Create a service check
 	var out struct{}
 	if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
 		t.Fatalf("err: %v", err)
@@ -117,8 +123,11 @@ func TestHealthServiceNodes(t *testing.T) {
 	defer srv.Shutdown()
 	defer srv.agent.Shutdown()
 
-	// Wait for a leader
-	time.Sleep(100 * time.Millisecond)
+	args := &structs.RegisterRequest{
+		Datacenter: "dc1",
+	}
+
+	testutil.WaitForLeader(t, srv.agent.RPC, args)
 
 	req, err := http.NewRequest("GET", "/v1/health/service/consul?dc=dc1", nil)
 	if err != nil {
@@ -145,10 +154,6 @@ func TestHealthServiceNodes_PassingFilter(t *testing.T) {
 	defer srv.Shutdown()
 	defer srv.agent.Shutdown()
 
-	// Wait for a leader
-	time.Sleep(100 * time.Millisecond)
-
-	// Create a failing service check
 	args := &structs.RegisterRequest{
 		Datacenter: "dc1",
 		Node:       srv.agent.config.NodeName,
@@ -160,6 +165,10 @@ func TestHealthServiceNodes_PassingFilter(t *testing.T) {
 			Status:    structs.HealthCritical,
 		},
 	}
+
+	testutil.WaitForLeader(t, srv.agent.RPC, args)
+
+	// Create a failing service check
 	var out struct{}
 	if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
 		t.Fatalf("err: %v", err)

From fcd5da3bf795bcb49b12d077b182003ca5a03183 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?William=20Tis=C3=A4ter?= <william@defunct.cc>
Date: Wed, 7 May 2014 23:17:00 +0200
Subject: [PATCH 09/39] Drop unnecessary sleep

---
 command/agent/http_test.go | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/command/agent/http_test.go b/command/agent/http_test.go
index 8d4a63d00411..60d28b6f466f 100644
--- a/command/agent/http_test.go
+++ b/command/agent/http_test.go
@@ -99,9 +99,6 @@ func TestContentTypeIsJSON(t *testing.T) {
 	defer srv.Shutdown()
 	defer srv.agent.Shutdown()
 
-	// Wait for a leader
-	time.Sleep(100 * time.Millisecond)
-
 	resp := httptest.NewRecorder()
 
 	handler := func(resp http.ResponseWriter, req *http.Request) (interface{}, error) {

From 0d90e480b6f013222b514b7c36060e055f490545 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?William=20Tis=C3=A4ter?= <william@defunct.cc>
Date: Wed, 7 May 2014 23:17:10 +0200
Subject: [PATCH 10/39] Remove all sleeps from `kvs_endpoint_test.go`

---
 command/agent/kvs_endpoint_test.go | 30 +++++++++++++++++++++---------
 1 file changed, 21 insertions(+), 9 deletions(-)

diff --git a/command/agent/kvs_endpoint_test.go b/command/agent/kvs_endpoint_test.go
index 8d3679e92cc7..97851573caa8 100644
--- a/command/agent/kvs_endpoint_test.go
+++ b/command/agent/kvs_endpoint_test.go
@@ -3,13 +3,13 @@ package agent
 import (
 	"bytes"
 	"fmt"
+	"github.com/hashicorp/consul/testutil"
 	"github.com/hashicorp/consul/consul/structs"
 	"net/http"
 	"net/http/httptest"
 	"os"
 	"reflect"
 	"testing"
-	"time"
 )
 
 func TestKVSEndpoint_PUT_GET_DELETE(t *testing.T) {
@@ -18,8 +18,11 @@ func TestKVSEndpoint_PUT_GET_DELETE(t *testing.T) {
 	defer srv.Shutdown()
 	defer srv.agent.Shutdown()
 
-	// Wait for a leader
-	time.Sleep(100 * time.Millisecond)
+	args := &structs.RegisterRequest{
+		Datacenter: "dc1",
+	}
+
+	testutil.WaitForLeader(t, srv.agent.RPC, args)
 
 	keys := []string{
 		"baz",
@@ -94,8 +97,11 @@ func TestKVSEndpoint_Recurse(t *testing.T) {
 	defer srv.Shutdown()
 	defer srv.agent.Shutdown()
 
-	// Wait for a leader
-	time.Sleep(100 * time.Millisecond)
+	args := &structs.RegisterRequest{
+		Datacenter: "dc1",
+	}
+
+	testutil.WaitForLeader(t, srv.agent.RPC, args)
 
 	keys := []string{
 		"bar",
@@ -191,8 +197,11 @@ func TestKVSEndpoint_CAS(t *testing.T) {
 	defer srv.Shutdown()
 	defer srv.agent.Shutdown()
 
-	// Wait for a leader
-	time.Sleep(100 * time.Millisecond)
+	args := &structs.RegisterRequest{
+		Datacenter: "dc1",
+	}
+
+	testutil.WaitForLeader(t, srv.agent.RPC, args)
 
 	{
 		buf := bytes.NewBuffer([]byte("test"))
@@ -289,8 +298,11 @@ func TestKVSEndpoint_ListKeys(t *testing.T) {
 	defer srv.Shutdown()
 	defer srv.agent.Shutdown()
 
-	// Wait for a leader
-	time.Sleep(100 * time.Millisecond)
+	args := &structs.RegisterRequest{
+		Datacenter: "dc1",
+	}
+
+	testutil.WaitForLeader(t, srv.agent.RPC, args)
 
 	keys := []string{
 		"bar",

From 4179b9dfe433457686a814e2ba1393244203492b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?William=20Tis=C3=A4ter?= <william@defunct.cc>
Date: Wed, 7 May 2014 23:27:32 +0200
Subject: [PATCH 11/39] Remove some sleeps from `local_test.go`

---
 command/agent/local_test.go | 21 +++++++++------------
 1 file changed, 9 insertions(+), 12 deletions(-)

diff --git a/command/agent/local_test.go b/command/agent/local_test.go
index a84c1a2929bd..8c3f8bda965f 100644
--- a/command/agent/local_test.go
+++ b/command/agent/local_test.go
@@ -1,6 +1,7 @@
 package agent
 
 import (
+	"github.com/hashicorp/consul/testutil"
 	"github.com/hashicorp/consul/consul/structs"
 	"os"
 	"reflect"
@@ -14,18 +15,16 @@ func TestAgentAntiEntropy_Services(t *testing.T) {
 	defer os.RemoveAll(dir)
 	defer agent.Shutdown()
 
-	// Wait for a leader
-	time.Sleep(100 * time.Millisecond)
-
-	// Register info
 	args := &structs.RegisterRequest{
 		Datacenter: "dc1",
 		Node:       agent.config.NodeName,
 		Address:    "127.0.0.1",
 	}
-	var out struct{}
 
-	// Exists both, same (noop)
+	testutil.WaitForLeader(t, agent.RPC, args)
+
+	// Register info. Exists both, same (noop)
+	var out struct{}
 	srv1 := &structs.NodeService{
 		ID:      "mysql",
 		Service: "mysql",
@@ -137,18 +136,16 @@ func TestAgentAntiEntropy_Checks(t *testing.T) {
 	defer os.RemoveAll(dir)
 	defer agent.Shutdown()
 
-	// Wait for a leader
-	time.Sleep(100 * time.Millisecond)
-
-	// Register info
 	args := &structs.RegisterRequest{
 		Datacenter: "dc1",
 		Node:       agent.config.NodeName,
 		Address:    "127.0.0.1",
 	}
-	var out struct{}
 
-	// Exists both, same (noop)
+	testutil.WaitForLeader(t, agent.RPC, args)
+
+	// Register info. Exists both, same (noop)
+	var out struct{}
 	chk1 := &structs.HealthCheck{
 		Node:    agent.config.NodeName,
 		CheckID: "mysql",

From c8168bd8fe7b08e8dd6013994f50a706eba1beb8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?William=20Tis=C3=A4ter?= <william@defunct.cc>
Date: Wed, 7 May 2014 23:41:14 +0200
Subject: [PATCH 12/39] Drop client arguments from `WaitForLeader`

---
 command/agent/catalog_endpoint_test.go | 42 +++++++++++++-------------
 command/agent/dns_test.go              | 22 +++++++-------
 command/agent/health_endpoint_test.go  | 22 +++-----------
 command/agent/kvs_endpoint_test.go     | 24 +++------------
 command/agent/local_test.go            |  4 +--
 consul/catalog_endpoint_test.go        |  4 +--
 consul/leader_test.go                  |  5 +--
 testutil/wait.go                       |  5 ++-
 8 files changed, 52 insertions(+), 76 deletions(-)

diff --git a/command/agent/catalog_endpoint_test.go b/command/agent/catalog_endpoint_test.go
index fa553d112a34..9ae11c4dbfd4 100644
--- a/command/agent/catalog_endpoint_test.go
+++ b/command/agent/catalog_endpoint_test.go
@@ -17,15 +17,15 @@ func TestCatalogRegister(t *testing.T) {
 	defer srv.Shutdown()
 	defer srv.agent.Shutdown()
 
+	testutil.WaitForLeader(t, srv.agent.RPC)
+
+	// Register node
 	args := &structs.RegisterRequest{
 		Datacenter: "dc1",
 		Node:       "foo",
 		Address:    "127.0.0.1",
 	}
 
-	testutil.WaitForLeader(t, srv.agent.RPC, args)
-
-	// Register node
 	req, err := http.NewRequest("GET", "/v1/catalog/register", nil)
 	if err != nil {
 		t.Fatalf("err: %v", err)
@@ -49,14 +49,14 @@ func TestCatalogDeregister(t *testing.T) {
 	defer srv.Shutdown()
 	defer srv.agent.Shutdown()
 
+	testutil.WaitForLeader(t, srv.agent.RPC)
+
+	// Register node
 	args := &structs.DeregisterRequest{
 		Datacenter: "dc1",
 		Node:       "foo",
 	}
 
-	testutil.WaitForLeader(t, srv.agent.RPC, args)
-
-	// Register node
 	req, err := http.NewRequest("GET", "/v1/catalog/deregister", nil)
 	if err != nil {
 		t.Fatalf("err: %v", err)
@@ -97,15 +97,15 @@ func TestCatalogNodes(t *testing.T) {
 	defer srv.Shutdown()
 	defer srv.agent.Shutdown()
 
+	testutil.WaitForLeader(t, srv.agent.RPC)
+
+	// Register node
 	args := &structs.RegisterRequest{
 		Datacenter: "dc1",
 		Node:       "foo",
 		Address:    "127.0.0.1",
 	}
 
-	testutil.WaitForLeader(t, srv.agent.RPC, args)
-
-	// Register node
 	var out struct{}
 	if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
 		t.Fatalf("err: %v", err)
@@ -137,13 +137,13 @@ func TestCatalogNodes_Blocking(t *testing.T) {
 	defer srv.Shutdown()
 	defer srv.agent.Shutdown()
 
+	testutil.WaitForLeader(t, srv.agent.RPC)
+
+	// Register node
 	args := &structs.DCSpecificRequest{
 		Datacenter: "dc1",
 	}
 
-	testutil.WaitForLeader(t, srv.agent.RPC, args)
-
-	// Register node
 	var out structs.IndexedNodes
 	if err := srv.agent.RPC("Catalog.ListNodes", *args, &out); err != nil {
 		t.Fatalf("err: %v", err)
@@ -198,6 +198,9 @@ func TestCatalogServices(t *testing.T) {
 	defer srv.Shutdown()
 	defer srv.agent.Shutdown()
 
+	testutil.WaitForLeader(t, srv.agent.RPC)
+
+	// Register node
 	args := &structs.RegisterRequest{
 		Datacenter: "dc1",
 		Node:       "foo",
@@ -207,9 +210,6 @@ func TestCatalogServices(t *testing.T) {
 		},
 	}
 
-	testutil.WaitForLeader(t, srv.agent.RPC, args)
-
-	// Register node
 	var out struct{}
 	if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
 		t.Fatalf("err: %v", err)
@@ -240,6 +240,9 @@ func TestCatalogServiceNodes(t *testing.T) {
 	defer srv.Shutdown()
 	defer srv.agent.Shutdown()
 
+	testutil.WaitForLeader(t, srv.agent.RPC)
+
+	// Register node
 	args := &structs.RegisterRequest{
 		Datacenter: "dc1",
 		Node:       "foo",
@@ -250,9 +253,6 @@ func TestCatalogServiceNodes(t *testing.T) {
 		},
 	}
 
-	testutil.WaitForLeader(t, srv.agent.RPC, args)
-
-	// Register node
 	var out struct{}
 	if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
 		t.Fatalf("err: %v", err)
@@ -283,6 +283,9 @@ func TestCatalogNodeServices(t *testing.T) {
 	defer srv.Shutdown()
 	defer srv.agent.Shutdown()
 
+	testutil.WaitForLeader(t, srv.agent.RPC)
+
+	// Register node
 	args := &structs.RegisterRequest{
 		Datacenter: "dc1",
 		Node:       "foo",
@@ -293,9 +296,6 @@ func TestCatalogNodeServices(t *testing.T) {
 		},
 	}
 
-	testutil.WaitForLeader(t, srv.agent.RPC, args)
-
-	// Register node
 	var out struct{}
 	if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
 		t.Fatalf("err: %v", err)
diff --git a/command/agent/dns_test.go b/command/agent/dns_test.go
index e874e3f649df..f1b0ba46a3aa 100644
--- a/command/agent/dns_test.go
+++ b/command/agent/dns_test.go
@@ -72,7 +72,7 @@ func TestDNS_NodeLookup(t *testing.T) {
 		Address:    "127.0.0.1",
 	}
 
-	testutil.WaitForLeader(t, srv.agent.RPC, args)
+	testutil.WaitForLeader(t, srv.agent.RPC)
 
 	var out struct{}
 	if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
@@ -136,7 +136,7 @@ func TestDNS_NodeLookup_PeriodName(t *testing.T) {
 		Address:    "127.0.0.1",
 	}
 
-	testutil.WaitForLeader(t, srv.agent.RPC, args)
+	testutil.WaitForLeader(t, srv.agent.RPC)
 
 	var out struct{}
 	if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
@@ -178,7 +178,7 @@ func TestDNS_NodeLookup_AAAA(t *testing.T) {
 		Address:    "::4242:4242",
 	}
 
-	testutil.WaitForLeader(t, srv.agent.RPC, args)
+	testutil.WaitForLeader(t, srv.agent.RPC)
 
 	var out struct{}
 	if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
@@ -220,7 +220,7 @@ func TestDNS_NodeLookup_CNAME(t *testing.T) {
 		Address:    "www.google.com",
 	}
 
-	testutil.WaitForLeader(t, srv.agent.RPC, args)
+	testutil.WaitForLeader(t, srv.agent.RPC)
 
 	var out struct{}
 	if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
@@ -268,7 +268,7 @@ func TestDNS_ServiceLookup(t *testing.T) {
 		},
 	}
 
-	testutil.WaitForLeader(t, srv.agent.RPC, args)
+	testutil.WaitForLeader(t, srv.agent.RPC)
 
 	var out struct{}
 	if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
@@ -329,7 +329,7 @@ func TestDNS_ServiceLookup_TagPeriod(t *testing.T) {
 		},
 	}
 
-	testutil.WaitForLeader(t, srv.agent.RPC, args)
+	testutil.WaitForLeader(t, srv.agent.RPC)
 
 	var out struct{}
 	if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
@@ -390,7 +390,7 @@ func TestDNS_ServiceLookup_Dedup(t *testing.T) {
 		},
 	}
 
-	testutil.WaitForLeader(t, srv.agent.RPC, args)
+	testutil.WaitForLeader(t, srv.agent.RPC)
 
 	var out struct{}
 	if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
@@ -467,7 +467,7 @@ func TestDNS_ServiceLookup_Dedup_SRV(t *testing.T) {
 		},
 	}
 
-	testutil.WaitForLeader(t, srv.agent.RPC, args)
+	testutil.WaitForLeader(t, srv.agent.RPC)
 
 	var out struct{}
 	if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
@@ -601,7 +601,7 @@ func TestDNS_ServiceLookup_FilterCritical(t *testing.T) {
 		},
 	}
 
-	testutil.WaitForLeader(t, srv.agent.RPC, args)
+	testutil.WaitForLeader(t, srv.agent.RPC)
 
 	var out struct{}
 	if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
@@ -660,7 +660,7 @@ func TestDNS_ServiceLookup_Randomize(t *testing.T) {
 			},
 		}
 
-		testutil.WaitForLeader(t, srv.agent.RPC, args)
+		testutil.WaitForLeader(t, srv.agent.RPC)
 
 		var out struct{}
 		if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
@@ -723,7 +723,7 @@ func TestDNS_ServiceLookup_CNAME(t *testing.T) {
 		},
 	}
 
-	testutil.WaitForLeader(t, srv.agent.RPC, args)
+	testutil.WaitForLeader(t, srv.agent.RPC)
 
 	var out struct{}
 	if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
diff --git a/command/agent/health_endpoint_test.go b/command/agent/health_endpoint_test.go
index 2f669846ca11..10f84be9d0b6 100644
--- a/command/agent/health_endpoint_test.go
+++ b/command/agent/health_endpoint_test.go
@@ -16,11 +16,7 @@ func TestHealthChecksInState(t *testing.T) {
 	defer srv.Shutdown()
 	defer srv.agent.Shutdown()
 
-	args := &structs.RegisterRequest{
-		Datacenter: "dc1",
-	}
-
-	testutil.WaitForLeader(t, srv.agent.RPC, args)
+	testutil.WaitForLeader(t, srv.agent.RPC)
 
 	req, err := http.NewRequest("GET", "/v1/health/state/passing?dc=dc1", nil)
 	if err != nil {
@@ -47,11 +43,7 @@ func TestHealthNodeChecks(t *testing.T) {
 	defer srv.Shutdown()
 	defer srv.agent.Shutdown()
 
-	args := &structs.RegisterRequest{
-		Datacenter: "dc1",
-	}
-
-	testutil.WaitForLeader(t, srv.agent.RPC, args)
+	testutil.WaitForLeader(t, srv.agent.RPC)
 
 	req, err := http.NewRequest("GET",
 		fmt.Sprintf("/v1/health/node/%s?dc=dc1", srv.agent.config.NodeName), nil)
@@ -90,7 +82,7 @@ func TestHealthServiceChecks(t *testing.T) {
 		},
 	}
 
-	testutil.WaitForLeader(t, srv.agent.RPC, args)
+	testutil.WaitForLeader(t, srv.agent.RPC)
 
 	// Create a service check
 	var out struct{}
@@ -123,11 +115,7 @@ func TestHealthServiceNodes(t *testing.T) {
 	defer srv.Shutdown()
 	defer srv.agent.Shutdown()
 
-	args := &structs.RegisterRequest{
-		Datacenter: "dc1",
-	}
-
-	testutil.WaitForLeader(t, srv.agent.RPC, args)
+	testutil.WaitForLeader(t, srv.agent.RPC)
 
 	req, err := http.NewRequest("GET", "/v1/health/service/consul?dc=dc1", nil)
 	if err != nil {
@@ -166,7 +154,7 @@ func TestHealthServiceNodes_PassingFilter(t *testing.T) {
 		},
 	}
 
-	testutil.WaitForLeader(t, srv.agent.RPC, args)
+	testutil.WaitForLeader(t, srv.agent.RPC)
 
 	// Create a failing service check
 	var out struct{}
diff --git a/command/agent/kvs_endpoint_test.go b/command/agent/kvs_endpoint_test.go
index 97851573caa8..3ccc12bdffae 100644
--- a/command/agent/kvs_endpoint_test.go
+++ b/command/agent/kvs_endpoint_test.go
@@ -18,11 +18,7 @@ func TestKVSEndpoint_PUT_GET_DELETE(t *testing.T) {
 	defer srv.Shutdown()
 	defer srv.agent.Shutdown()
 
-	args := &structs.RegisterRequest{
-		Datacenter: "dc1",
-	}
-
-	testutil.WaitForLeader(t, srv.agent.RPC, args)
+	testutil.WaitForLeader(t, srv.agent.RPC)
 
 	keys := []string{
 		"baz",
@@ -97,11 +93,7 @@ func TestKVSEndpoint_Recurse(t *testing.T) {
 	defer srv.Shutdown()
 	defer srv.agent.Shutdown()
 
-	args := &structs.RegisterRequest{
-		Datacenter: "dc1",
-	}
-
-	testutil.WaitForLeader(t, srv.agent.RPC, args)
+	testutil.WaitForLeader(t, srv.agent.RPC)
 
 	keys := []string{
 		"bar",
@@ -197,11 +189,7 @@ func TestKVSEndpoint_CAS(t *testing.T) {
 	defer srv.Shutdown()
 	defer srv.agent.Shutdown()
 
-	args := &structs.RegisterRequest{
-		Datacenter: "dc1",
-	}
-
-	testutil.WaitForLeader(t, srv.agent.RPC, args)
+	testutil.WaitForLeader(t, srv.agent.RPC)
 
 	{
 		buf := bytes.NewBuffer([]byte("test"))
@@ -298,11 +286,7 @@ func TestKVSEndpoint_ListKeys(t *testing.T) {
 	defer srv.Shutdown()
 	defer srv.agent.Shutdown()
 
-	args := &structs.RegisterRequest{
-		Datacenter: "dc1",
-	}
-
-	testutil.WaitForLeader(t, srv.agent.RPC, args)
+	testutil.WaitForLeader(t, srv.agent.RPC)
 
 	keys := []string{
 		"bar",
diff --git a/command/agent/local_test.go b/command/agent/local_test.go
index 8c3f8bda965f..69a85a8e5e34 100644
--- a/command/agent/local_test.go
+++ b/command/agent/local_test.go
@@ -21,7 +21,7 @@ func TestAgentAntiEntropy_Services(t *testing.T) {
 		Address:    "127.0.0.1",
 	}
 
-	testutil.WaitForLeader(t, agent.RPC, args)
+	testutil.WaitForLeader(t, agent.RPC)
 
 	// Register info. Exists both, same (noop)
 	var out struct{}
@@ -142,7 +142,7 @@ func TestAgentAntiEntropy_Checks(t *testing.T) {
 		Address:    "127.0.0.1",
 	}
 
-	testutil.WaitForLeader(t, agent.RPC, args)
+	testutil.WaitForLeader(t, agent.RPC)
 
 	// Register info. Exists both, same (noop)
 	var out struct{}
diff --git a/consul/catalog_endpoint_test.go b/consul/catalog_endpoint_test.go
index 3f01b49db66d..66059b5e756c 100644
--- a/consul/catalog_endpoint_test.go
+++ b/consul/catalog_endpoint_test.go
@@ -259,8 +259,8 @@ func TestCatalogListNodes_StaleRaad(t *testing.T) {
 		QueryOptions: structs.QueryOptions{AllowStale: true},
 	}
 
-	testutil.WaitForLeader(t, client1.Call, args)
-	testutil.WaitForLeader(t, client2.Call, args)
+	testutil.WaitForLeader(t, client1.Call)
+	testutil.WaitForLeader(t, client2.Call)
 
 	// Use the follower as the client
 	var client *rpc.Client
diff --git a/consul/leader_test.go b/consul/leader_test.go
index f1ea3b0db93d..09e1856e515f 100644
--- a/consul/leader_test.go
+++ b/consul/leader_test.go
@@ -2,6 +2,7 @@ package consul
 
 import (
 	"fmt"
+	"github.com/hashicorp/consul/testutil"
 	"github.com/hashicorp/consul/consul/structs"
 	"github.com/hashicorp/serf/serf"
 	"os"
@@ -18,8 +19,8 @@ func TestLeader_RegisterMember(t *testing.T) {
 	defer os.RemoveAll(dir2)
 	defer c1.Shutdown()
 
-	// Wait until we have a leader
-	time.Sleep(100 * time.Millisecond)
+	client := rpcClient(t, s1)
+	testutil.WaitForLeader(t, client.Call)
 
 	// Try to join
 	addr := fmt.Sprintf("127.0.0.1:%d",
diff --git a/testutil/wait.go b/testutil/wait.go
index 78ef2d140cc3..20ab8ea5e5c1 100644
--- a/testutil/wait.go
+++ b/testutil/wait.go
@@ -29,9 +29,12 @@ func WaitForResult(test testFn, error errorFn) {
 
 type rpcFn func(string, interface {}, interface {}) error
 
-func WaitForLeader(t *testing.T, rpc rpcFn, args interface{}) structs.IndexedNodes {
+func WaitForLeader(t *testing.T, rpc rpcFn) structs.IndexedNodes {
 	var out structs.IndexedNodes
 	WaitForResult(func() (bool, error) {
+		args := &structs.RegisterRequest{
+			Datacenter: "dc1",
+		}
 		err := rpc("Catalog.ListNodes", args, &out)
 		return out.QueryMeta.KnownLeader, err
 	}, func(err error) {

From 0ed5599ffed451db09c5df95e9b227a5f3c0b80e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?William=20Tis=C3=A4ter?= <william@defunct.cc>
Date: Wed, 7 May 2014 23:42:15 +0200
Subject: [PATCH 13/39] Add test file for testutil

---
 testutil/wait_test.go | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 testutil/wait_test.go

diff --git a/testutil/wait_test.go b/testutil/wait_test.go
new file mode 100644
index 000000000000..110b2e6a7979
--- /dev/null
+++ b/testutil/wait_test.go
@@ -0,0 +1 @@
+package testutil

From 93c7103bb54bcc0c26ff80a05bd9b37d03c6cf96 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?William=20Tis=C3=A4ter?= <william@defunct.cc>
Date: Wed, 7 May 2014 23:47:16 +0200
Subject: [PATCH 14/39] Move `WaitForLeader` calls to top of test

---
 command/agent/dns_test.go             | 44 +++++++++++++--------------
 command/agent/health_endpoint_test.go | 12 ++++----
 command/agent/local_test.go           | 14 +++++----
 consul/catalog_endpoint_test.go       |  6 ++--
 4 files changed, 39 insertions(+), 37 deletions(-)

diff --git a/command/agent/dns_test.go b/command/agent/dns_test.go
index f1b0ba46a3aa..1aec2f2a0a70 100644
--- a/command/agent/dns_test.go
+++ b/command/agent/dns_test.go
@@ -65,6 +65,8 @@ func TestDNS_NodeLookup(t *testing.T) {
 	defer os.RemoveAll(dir)
 	defer srv.agent.Shutdown()
 
+	testutil.WaitForLeader(t, srv.agent.RPC)
+
 	// Register node
 	args := &structs.RegisterRequest{
 		Datacenter: "dc1",
@@ -72,8 +74,6 @@ func TestDNS_NodeLookup(t *testing.T) {
 		Address:    "127.0.0.1",
 	}
 
-	testutil.WaitForLeader(t, srv.agent.RPC)
-
 	var out struct{}
 	if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
 		t.Fatalf("err: %v", err)
@@ -129,6 +129,8 @@ func TestDNS_NodeLookup_PeriodName(t *testing.T) {
 	defer os.RemoveAll(dir)
 	defer srv.agent.Shutdown()
 
+	testutil.WaitForLeader(t, srv.agent.RPC)
+
 	// Register node with period in name
 	args := &structs.RegisterRequest{
 		Datacenter: "dc1",
@@ -136,8 +138,6 @@ func TestDNS_NodeLookup_PeriodName(t *testing.T) {
 		Address:    "127.0.0.1",
 	}
 
-	testutil.WaitForLeader(t, srv.agent.RPC)
-
 	var out struct{}
 	if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
 		t.Fatalf("err: %v", err)
@@ -171,6 +171,8 @@ func TestDNS_NodeLookup_AAAA(t *testing.T) {
 	defer os.RemoveAll(dir)
 	defer srv.agent.Shutdown()
 
+	testutil.WaitForLeader(t, srv.agent.RPC)
+
 	// Register node
 	args := &structs.RegisterRequest{
 		Datacenter: "dc1",
@@ -178,8 +180,6 @@ func TestDNS_NodeLookup_AAAA(t *testing.T) {
 		Address:    "::4242:4242",
 	}
 
-	testutil.WaitForLeader(t, srv.agent.RPC)
-
 	var out struct{}
 	if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
 		t.Fatalf("err: %v", err)
@@ -213,6 +213,8 @@ func TestDNS_NodeLookup_CNAME(t *testing.T) {
 	defer os.RemoveAll(dir)
 	defer srv.agent.Shutdown()
 
+	testutil.WaitForLeader(t, srv.agent.RPC)
+
 	// Register node
 	args := &structs.RegisterRequest{
 		Datacenter: "dc1",
@@ -220,8 +222,6 @@ func TestDNS_NodeLookup_CNAME(t *testing.T) {
 		Address:    "www.google.com",
 	}
 
-	testutil.WaitForLeader(t, srv.agent.RPC)
-
 	var out struct{}
 	if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
 		t.Fatalf("err: %v", err)
@@ -256,6 +256,8 @@ func TestDNS_ServiceLookup(t *testing.T) {
 	defer os.RemoveAll(dir)
 	defer srv.agent.Shutdown()
 
+	testutil.WaitForLeader(t, srv.agent.RPC)
+
 	// Register node
 	args := &structs.RegisterRequest{
 		Datacenter: "dc1",
@@ -268,8 +270,6 @@ func TestDNS_ServiceLookup(t *testing.T) {
 		},
 	}
 
-	testutil.WaitForLeader(t, srv.agent.RPC)
-
 	var out struct{}
 	if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
 		t.Fatalf("err: %v", err)
@@ -317,6 +317,8 @@ func TestDNS_ServiceLookup_TagPeriod(t *testing.T) {
 	defer os.RemoveAll(dir)
 	defer srv.agent.Shutdown()
 
+	testutil.WaitForLeader(t, srv.agent.RPC)
+
 	// Register node
 	args := &structs.RegisterRequest{
 		Datacenter: "dc1",
@@ -329,8 +331,6 @@ func TestDNS_ServiceLookup_TagPeriod(t *testing.T) {
 		},
 	}
 
-	testutil.WaitForLeader(t, srv.agent.RPC)
-
 	var out struct{}
 	if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
 		t.Fatalf("err: %v", err)
@@ -378,6 +378,8 @@ func TestDNS_ServiceLookup_Dedup(t *testing.T) {
 	defer os.RemoveAll(dir)
 	defer srv.agent.Shutdown()
 
+	testutil.WaitForLeader(t, srv.agent.RPC)
+
 	// Register node
 	args := &structs.RegisterRequest{
 		Datacenter: "dc1",
@@ -390,8 +392,6 @@ func TestDNS_ServiceLookup_Dedup(t *testing.T) {
 		},
 	}
 
-	testutil.WaitForLeader(t, srv.agent.RPC)
-
 	var out struct{}
 	if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
 		t.Fatalf("err: %v", err)
@@ -455,6 +455,8 @@ func TestDNS_ServiceLookup_Dedup_SRV(t *testing.T) {
 	defer os.RemoveAll(dir)
 	defer srv.agent.Shutdown()
 
+	testutil.WaitForLeader(t, srv.agent.RPC)
+
 	// Register node
 	args := &structs.RegisterRequest{
 		Datacenter: "dc1",
@@ -467,8 +469,6 @@ func TestDNS_ServiceLookup_Dedup_SRV(t *testing.T) {
 		},
 	}
 
-	testutil.WaitForLeader(t, srv.agent.RPC)
-
 	var out struct{}
 	if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
 		t.Fatalf("err: %v", err)
@@ -584,6 +584,8 @@ func TestDNS_ServiceLookup_FilterCritical(t *testing.T) {
 	defer os.RemoveAll(dir)
 	defer srv.agent.Shutdown()
 
+	testutil.WaitForLeader(t, srv.agent.RPC)
+
 	// Register nodes
 	args := &structs.RegisterRequest{
 		Datacenter: "dc1",
@@ -601,8 +603,6 @@ func TestDNS_ServiceLookup_FilterCritical(t *testing.T) {
 		},
 	}
 
-	testutil.WaitForLeader(t, srv.agent.RPC)
-
 	var out struct{}
 	if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
 		t.Fatalf("err: %v", err)
@@ -648,6 +648,8 @@ func TestDNS_ServiceLookup_Randomize(t *testing.T) {
 	defer os.RemoveAll(dir)
 	defer srv.agent.Shutdown()
 
+	testutil.WaitForLeader(t, srv.agent.RPC)
+
 	// Register nodes
 	for i := 0; i < 3*maxServiceResponses; i++ {
 		args := &structs.RegisterRequest{
@@ -660,8 +662,6 @@ func TestDNS_ServiceLookup_Randomize(t *testing.T) {
 			},
 		}
 
-		testutil.WaitForLeader(t, srv.agent.RPC)
-
 		var out struct{}
 		if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
 			t.Fatalf("err: %v", err)
@@ -712,6 +712,8 @@ func TestDNS_ServiceLookup_CNAME(t *testing.T) {
 	defer os.RemoveAll(dir)
 	defer srv.agent.Shutdown()
 
+	testutil.WaitForLeader(t, srv.agent.RPC)
+
 	// Register node
 	args := &structs.RegisterRequest{
 		Datacenter: "dc1",
@@ -723,8 +725,6 @@ func TestDNS_ServiceLookup_CNAME(t *testing.T) {
 		},
 	}
 
-	testutil.WaitForLeader(t, srv.agent.RPC)
-
 	var out struct{}
 	if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
 		t.Fatalf("err: %v", err)
diff --git a/command/agent/health_endpoint_test.go b/command/agent/health_endpoint_test.go
index 10f84be9d0b6..4a097ae65d3c 100644
--- a/command/agent/health_endpoint_test.go
+++ b/command/agent/health_endpoint_test.go
@@ -71,6 +71,9 @@ func TestHealthServiceChecks(t *testing.T) {
 	defer srv.Shutdown()
 	defer srv.agent.Shutdown()
 
+	testutil.WaitForLeader(t, srv.agent.RPC)
+
+	// Create a service check
 	args := &structs.RegisterRequest{
 		Datacenter: "dc1",
 		Node:       srv.agent.config.NodeName,
@@ -82,9 +85,6 @@ func TestHealthServiceChecks(t *testing.T) {
 		},
 	}
 
-	testutil.WaitForLeader(t, srv.agent.RPC)
-
-	// Create a service check
 	var out struct{}
 	if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
 		t.Fatalf("err: %v", err)
@@ -142,6 +142,9 @@ func TestHealthServiceNodes_PassingFilter(t *testing.T) {
 	defer srv.Shutdown()
 	defer srv.agent.Shutdown()
 
+	testutil.WaitForLeader(t, srv.agent.RPC)
+
+	// Create a failing service check
 	args := &structs.RegisterRequest{
 		Datacenter: "dc1",
 		Node:       srv.agent.config.NodeName,
@@ -154,9 +157,6 @@ func TestHealthServiceNodes_PassingFilter(t *testing.T) {
 		},
 	}
 
-	testutil.WaitForLeader(t, srv.agent.RPC)
-
-	// Create a failing service check
 	var out struct{}
 	if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
 		t.Fatalf("err: %v", err)
diff --git a/command/agent/local_test.go b/command/agent/local_test.go
index 69a85a8e5e34..2eed35d68dbe 100644
--- a/command/agent/local_test.go
+++ b/command/agent/local_test.go
@@ -15,15 +15,16 @@ func TestAgentAntiEntropy_Services(t *testing.T) {
 	defer os.RemoveAll(dir)
 	defer agent.Shutdown()
 
+	testutil.WaitForLeader(t, agent.RPC)
+
+	// Register info
 	args := &structs.RegisterRequest{
 		Datacenter: "dc1",
 		Node:       agent.config.NodeName,
 		Address:    "127.0.0.1",
 	}
 
-	testutil.WaitForLeader(t, agent.RPC)
-
-	// Register info. Exists both, same (noop)
+	// Exists both, same (noop)
 	var out struct{}
 	srv1 := &structs.NodeService{
 		ID:      "mysql",
@@ -136,15 +137,16 @@ func TestAgentAntiEntropy_Checks(t *testing.T) {
 	defer os.RemoveAll(dir)
 	defer agent.Shutdown()
 
+	testutil.WaitForLeader(t, agent.RPC)
+
+	// Register info
 	args := &structs.RegisterRequest{
 		Datacenter: "dc1",
 		Node:       agent.config.NodeName,
 		Address:    "127.0.0.1",
 	}
 
-	testutil.WaitForLeader(t, agent.RPC)
-
-	// Register info. Exists both, same (noop)
+	// Exists both, same (noop)
 	var out struct{}
 	chk1 := &structs.HealthCheck{
 		Node:    agent.config.NodeName,
diff --git a/consul/catalog_endpoint_test.go b/consul/catalog_endpoint_test.go
index 66059b5e756c..4c5a2c70b493 100644
--- a/consul/catalog_endpoint_test.go
+++ b/consul/catalog_endpoint_test.go
@@ -254,14 +254,14 @@ func TestCatalogListNodes_StaleRaad(t *testing.T) {
 		t.Fatalf("err: %v", err)
 	}
 
+	testutil.WaitForLeader(t, client1.Call)
+	testutil.WaitForLeader(t, client2.Call)
+
 	args := structs.DCSpecificRequest{
 		Datacenter:   "dc1",
 		QueryOptions: structs.QueryOptions{AllowStale: true},
 	}
 
-	testutil.WaitForLeader(t, client1.Call)
-	testutil.WaitForLeader(t, client2.Call)
-
 	// Use the follower as the client
 	var client *rpc.Client
 	if !s1.IsLeader() {

From 05243b35cd08610b957a4ca1eb4ef862bb140145 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?William=20Tis=C3=A4ter?= <william@defunct.cc>
Date: Wed, 7 May 2014 23:51:15 +0200
Subject: [PATCH 15/39] Move argument structs back in original position

---
 command/agent/catalog_endpoint_test.go | 18 +++++++-----------
 consul/catalog_endpoint_test.go        |  9 ++++-----
 2 files changed, 11 insertions(+), 16 deletions(-)

diff --git a/command/agent/catalog_endpoint_test.go b/command/agent/catalog_endpoint_test.go
index 9ae11c4dbfd4..c9e094dcb8ce 100644
--- a/command/agent/catalog_endpoint_test.go
+++ b/command/agent/catalog_endpoint_test.go
@@ -20,16 +20,14 @@ func TestCatalogRegister(t *testing.T) {
 	testutil.WaitForLeader(t, srv.agent.RPC)
 
 	// Register node
-	args := &structs.RegisterRequest{
-		Datacenter: "dc1",
-		Node:       "foo",
-		Address:    "127.0.0.1",
-	}
-
 	req, err := http.NewRequest("GET", "/v1/catalog/register", nil)
 	if err != nil {
 		t.Fatalf("err: %v", err)
 	}
+	args := &structs.RegisterRequest{
+		Node:    "foo",
+		Address: "127.0.0.1",
+	}
 	req.Body = encodeReq(args)
 
 	obj, err := srv.CatalogRegister(nil, req)
@@ -52,15 +50,13 @@ func TestCatalogDeregister(t *testing.T) {
 	testutil.WaitForLeader(t, srv.agent.RPC)
 
 	// Register node
-	args := &structs.DeregisterRequest{
-		Datacenter: "dc1",
-		Node:       "foo",
-	}
-
 	req, err := http.NewRequest("GET", "/v1/catalog/deregister", nil)
 	if err != nil {
 		t.Fatalf("err: %v", err)
 	}
+	args := &structs.DeregisterRequest{
+		Node: "foo",
+	}
 	req.Body = encodeReq(args)
 
 	obj, err := srv.CatalogDeregister(nil, req)
diff --git a/consul/catalog_endpoint_test.go b/consul/catalog_endpoint_test.go
index 4c5a2c70b493..149651de6b33 100644
--- a/consul/catalog_endpoint_test.go
+++ b/consul/catalog_endpoint_test.go
@@ -257,11 +257,6 @@ func TestCatalogListNodes_StaleRaad(t *testing.T) {
 	testutil.WaitForLeader(t, client1.Call)
 	testutil.WaitForLeader(t, client2.Call)
 
-	args := structs.DCSpecificRequest{
-		Datacenter:   "dc1",
-		QueryOptions: structs.QueryOptions{AllowStale: true},
-	}
-
 	// Use the follower as the client
 	var client *rpc.Client
 	if !s1.IsLeader() {
@@ -276,6 +271,10 @@ func TestCatalogListNodes_StaleRaad(t *testing.T) {
 		s2.fsm.State().EnsureNode(1, structs.Node{"foo", "127.0.0.1"})
 	}
 
+	args := structs.DCSpecificRequest{
+		Datacenter:   "dc1",
+		QueryOptions: structs.QueryOptions{AllowStale: true},
+	}
 	var out structs.IndexedNodes
 	if err := client.Call("Catalog.ListNodes", &args, &out); err != nil {
 		t.Fatalf("err: %v", err)

From 78145ecfd5a27118d73de789b1859b8224fe818c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?William=20Tisa=CC=88ter?= <william@defunct.cc>
Date: Fri, 9 May 2014 01:17:35 +0200
Subject: [PATCH 16/39] Require datacenter argument for `WaitForLeader`

---
 command/agent/catalog_endpoint_test.go | 14 +++++++-------
 command/agent/dns_test.go              | 22 +++++++++++-----------
 command/agent/health_endpoint_test.go  | 10 +++++-----
 command/agent/kvs_endpoint_test.go     |  8 ++++----
 command/agent/local_test.go            |  4 ++--
 consul/leader_test.go                  |  2 +-
 testutil/wait.go                       |  4 ++--
 7 files changed, 32 insertions(+), 32 deletions(-)

diff --git a/command/agent/catalog_endpoint_test.go b/command/agent/catalog_endpoint_test.go
index c9e094dcb8ce..d9454ab7ed87 100644
--- a/command/agent/catalog_endpoint_test.go
+++ b/command/agent/catalog_endpoint_test.go
@@ -17,7 +17,7 @@ func TestCatalogRegister(t *testing.T) {
 	defer srv.Shutdown()
 	defer srv.agent.Shutdown()
 
-	testutil.WaitForLeader(t, srv.agent.RPC)
+	testutil.WaitForLeader(t, srv.agent.RPC, "dc1")
 
 	// Register node
 	req, err := http.NewRequest("GET", "/v1/catalog/register", nil)
@@ -47,7 +47,7 @@ func TestCatalogDeregister(t *testing.T) {
 	defer srv.Shutdown()
 	defer srv.agent.Shutdown()
 
-	testutil.WaitForLeader(t, srv.agent.RPC)
+	testutil.WaitForLeader(t, srv.agent.RPC, "dc1")
 
 	// Register node
 	req, err := http.NewRequest("GET", "/v1/catalog/deregister", nil)
@@ -93,7 +93,7 @@ func TestCatalogNodes(t *testing.T) {
 	defer srv.Shutdown()
 	defer srv.agent.Shutdown()
 
-	testutil.WaitForLeader(t, srv.agent.RPC)
+	testutil.WaitForLeader(t, srv.agent.RPC, "dc1")
 
 	// Register node
 	args := &structs.RegisterRequest{
@@ -133,7 +133,7 @@ func TestCatalogNodes_Blocking(t *testing.T) {
 	defer srv.Shutdown()
 	defer srv.agent.Shutdown()
 
-	testutil.WaitForLeader(t, srv.agent.RPC)
+	testutil.WaitForLeader(t, srv.agent.RPC, "dc1")
 
 	// Register node
 	args := &structs.DCSpecificRequest{
@@ -194,7 +194,7 @@ func TestCatalogServices(t *testing.T) {
 	defer srv.Shutdown()
 	defer srv.agent.Shutdown()
 
-	testutil.WaitForLeader(t, srv.agent.RPC)
+	testutil.WaitForLeader(t, srv.agent.RPC, "dc1")
 
 	// Register node
 	args := &structs.RegisterRequest{
@@ -236,7 +236,7 @@ func TestCatalogServiceNodes(t *testing.T) {
 	defer srv.Shutdown()
 	defer srv.agent.Shutdown()
 
-	testutil.WaitForLeader(t, srv.agent.RPC)
+	testutil.WaitForLeader(t, srv.agent.RPC, "dc1")
 
 	// Register node
 	args := &structs.RegisterRequest{
@@ -279,7 +279,7 @@ func TestCatalogNodeServices(t *testing.T) {
 	defer srv.Shutdown()
 	defer srv.agent.Shutdown()
 
-	testutil.WaitForLeader(t, srv.agent.RPC)
+	testutil.WaitForLeader(t, srv.agent.RPC, "dc1")
 
 	// Register node
 	args := &structs.RegisterRequest{
diff --git a/command/agent/dns_test.go b/command/agent/dns_test.go
index 1aec2f2a0a70..eeab068d9d5e 100644
--- a/command/agent/dns_test.go
+++ b/command/agent/dns_test.go
@@ -65,7 +65,7 @@ func TestDNS_NodeLookup(t *testing.T) {
 	defer os.RemoveAll(dir)
 	defer srv.agent.Shutdown()
 
-	testutil.WaitForLeader(t, srv.agent.RPC)
+	testutil.WaitForLeader(t, srv.agent.RPC, "dc1")
 
 	// Register node
 	args := &structs.RegisterRequest{
@@ -129,7 +129,7 @@ func TestDNS_NodeLookup_PeriodName(t *testing.T) {
 	defer os.RemoveAll(dir)
 	defer srv.agent.Shutdown()
 
-	testutil.WaitForLeader(t, srv.agent.RPC)
+	testutil.WaitForLeader(t, srv.agent.RPC, "dc1")
 
 	// Register node with period in name
 	args := &structs.RegisterRequest{
@@ -171,7 +171,7 @@ func TestDNS_NodeLookup_AAAA(t *testing.T) {
 	defer os.RemoveAll(dir)
 	defer srv.agent.Shutdown()
 
-	testutil.WaitForLeader(t, srv.agent.RPC)
+	testutil.WaitForLeader(t, srv.agent.RPC, "dc1")
 
 	// Register node
 	args := &structs.RegisterRequest{
@@ -213,7 +213,7 @@ func TestDNS_NodeLookup_CNAME(t *testing.T) {
 	defer os.RemoveAll(dir)
 	defer srv.agent.Shutdown()
 
-	testutil.WaitForLeader(t, srv.agent.RPC)
+	testutil.WaitForLeader(t, srv.agent.RPC, "dc1")
 
 	// Register node
 	args := &structs.RegisterRequest{
@@ -256,7 +256,7 @@ func TestDNS_ServiceLookup(t *testing.T) {
 	defer os.RemoveAll(dir)
 	defer srv.agent.Shutdown()
 
-	testutil.WaitForLeader(t, srv.agent.RPC)
+	testutil.WaitForLeader(t, srv.agent.RPC, "dc1")
 
 	// Register node
 	args := &structs.RegisterRequest{
@@ -317,7 +317,7 @@ func TestDNS_ServiceLookup_TagPeriod(t *testing.T) {
 	defer os.RemoveAll(dir)
 	defer srv.agent.Shutdown()
 
-	testutil.WaitForLeader(t, srv.agent.RPC)
+	testutil.WaitForLeader(t, srv.agent.RPC, "dc1")
 
 	// Register node
 	args := &structs.RegisterRequest{
@@ -378,7 +378,7 @@ func TestDNS_ServiceLookup_Dedup(t *testing.T) {
 	defer os.RemoveAll(dir)
 	defer srv.agent.Shutdown()
 
-	testutil.WaitForLeader(t, srv.agent.RPC)
+	testutil.WaitForLeader(t, srv.agent.RPC, "dc1")
 
 	// Register node
 	args := &structs.RegisterRequest{
@@ -455,7 +455,7 @@ func TestDNS_ServiceLookup_Dedup_SRV(t *testing.T) {
 	defer os.RemoveAll(dir)
 	defer srv.agent.Shutdown()
 
-	testutil.WaitForLeader(t, srv.agent.RPC)
+	testutil.WaitForLeader(t, srv.agent.RPC, "dc1")
 
 	// Register node
 	args := &structs.RegisterRequest{
@@ -584,7 +584,7 @@ func TestDNS_ServiceLookup_FilterCritical(t *testing.T) {
 	defer os.RemoveAll(dir)
 	defer srv.agent.Shutdown()
 
-	testutil.WaitForLeader(t, srv.agent.RPC)
+	testutil.WaitForLeader(t, srv.agent.RPC, "dc1")
 
 	// Register nodes
 	args := &structs.RegisterRequest{
@@ -648,7 +648,7 @@ func TestDNS_ServiceLookup_Randomize(t *testing.T) {
 	defer os.RemoveAll(dir)
 	defer srv.agent.Shutdown()
 
-	testutil.WaitForLeader(t, srv.agent.RPC)
+	testutil.WaitForLeader(t, srv.agent.RPC, "dc1")
 
 	// Register nodes
 	for i := 0; i < 3*maxServiceResponses; i++ {
@@ -712,7 +712,7 @@ func TestDNS_ServiceLookup_CNAME(t *testing.T) {
 	defer os.RemoveAll(dir)
 	defer srv.agent.Shutdown()
 
-	testutil.WaitForLeader(t, srv.agent.RPC)
+	testutil.WaitForLeader(t, srv.agent.RPC, "dc1")
 
 	// Register node
 	args := &structs.RegisterRequest{
diff --git a/command/agent/health_endpoint_test.go b/command/agent/health_endpoint_test.go
index 4a097ae65d3c..910d37df8b10 100644
--- a/command/agent/health_endpoint_test.go
+++ b/command/agent/health_endpoint_test.go
@@ -16,7 +16,7 @@ func TestHealthChecksInState(t *testing.T) {
 	defer srv.Shutdown()
 	defer srv.agent.Shutdown()
 
-	testutil.WaitForLeader(t, srv.agent.RPC)
+	testutil.WaitForLeader(t, srv.agent.RPC, "dc1")
 
 	req, err := http.NewRequest("GET", "/v1/health/state/passing?dc=dc1", nil)
 	if err != nil {
@@ -43,7 +43,7 @@ func TestHealthNodeChecks(t *testing.T) {
 	defer srv.Shutdown()
 	defer srv.agent.Shutdown()
 
-	testutil.WaitForLeader(t, srv.agent.RPC)
+	testutil.WaitForLeader(t, srv.agent.RPC, "dc1")
 
 	req, err := http.NewRequest("GET",
 		fmt.Sprintf("/v1/health/node/%s?dc=dc1", srv.agent.config.NodeName), nil)
@@ -71,7 +71,7 @@ func TestHealthServiceChecks(t *testing.T) {
 	defer srv.Shutdown()
 	defer srv.agent.Shutdown()
 
-	testutil.WaitForLeader(t, srv.agent.RPC)
+	testutil.WaitForLeader(t, srv.agent.RPC, "dc1")
 
 	// Create a service check
 	args := &structs.RegisterRequest{
@@ -115,7 +115,7 @@ func TestHealthServiceNodes(t *testing.T) {
 	defer srv.Shutdown()
 	defer srv.agent.Shutdown()
 
-	testutil.WaitForLeader(t, srv.agent.RPC)
+	testutil.WaitForLeader(t, srv.agent.RPC, "dc1")
 
 	req, err := http.NewRequest("GET", "/v1/health/service/consul?dc=dc1", nil)
 	if err != nil {
@@ -142,7 +142,7 @@ func TestHealthServiceNodes_PassingFilter(t *testing.T) {
 	defer srv.Shutdown()
 	defer srv.agent.Shutdown()
 
-	testutil.WaitForLeader(t, srv.agent.RPC)
+	testutil.WaitForLeader(t, srv.agent.RPC, "dc1")
 
 	// Create a failing service check
 	args := &structs.RegisterRequest{
diff --git a/command/agent/kvs_endpoint_test.go b/command/agent/kvs_endpoint_test.go
index 3ccc12bdffae..61216faafca2 100644
--- a/command/agent/kvs_endpoint_test.go
+++ b/command/agent/kvs_endpoint_test.go
@@ -18,7 +18,7 @@ func TestKVSEndpoint_PUT_GET_DELETE(t *testing.T) {
 	defer srv.Shutdown()
 	defer srv.agent.Shutdown()
 
-	testutil.WaitForLeader(t, srv.agent.RPC)
+	testutil.WaitForLeader(t, srv.agent.RPC, "dc1")
 
 	keys := []string{
 		"baz",
@@ -93,7 +93,7 @@ func TestKVSEndpoint_Recurse(t *testing.T) {
 	defer srv.Shutdown()
 	defer srv.agent.Shutdown()
 
-	testutil.WaitForLeader(t, srv.agent.RPC)
+	testutil.WaitForLeader(t, srv.agent.RPC, "dc1")
 
 	keys := []string{
 		"bar",
@@ -189,7 +189,7 @@ func TestKVSEndpoint_CAS(t *testing.T) {
 	defer srv.Shutdown()
 	defer srv.agent.Shutdown()
 
-	testutil.WaitForLeader(t, srv.agent.RPC)
+	testutil.WaitForLeader(t, srv.agent.RPC, "dc1")
 
 	{
 		buf := bytes.NewBuffer([]byte("test"))
@@ -286,7 +286,7 @@ func TestKVSEndpoint_ListKeys(t *testing.T) {
 	defer srv.Shutdown()
 	defer srv.agent.Shutdown()
 
-	testutil.WaitForLeader(t, srv.agent.RPC)
+	testutil.WaitForLeader(t, srv.agent.RPC, "dc1")
 
 	keys := []string{
 		"bar",
diff --git a/command/agent/local_test.go b/command/agent/local_test.go
index 2eed35d68dbe..9ce16fed31f6 100644
--- a/command/agent/local_test.go
+++ b/command/agent/local_test.go
@@ -15,7 +15,7 @@ func TestAgentAntiEntropy_Services(t *testing.T) {
 	defer os.RemoveAll(dir)
 	defer agent.Shutdown()
 
-	testutil.WaitForLeader(t, agent.RPC)
+	testutil.WaitForLeader(t, agent.RPC, "dc1")
 
 	// Register info
 	args := &structs.RegisterRequest{
@@ -137,7 +137,7 @@ func TestAgentAntiEntropy_Checks(t *testing.T) {
 	defer os.RemoveAll(dir)
 	defer agent.Shutdown()
 
-	testutil.WaitForLeader(t, agent.RPC)
+	testutil.WaitForLeader(t, agent.RPC, "dc1")
 
 	// Register info
 	args := &structs.RegisterRequest{
diff --git a/consul/leader_test.go b/consul/leader_test.go
index 09e1856e515f..f5b56174ebf9 100644
--- a/consul/leader_test.go
+++ b/consul/leader_test.go
@@ -20,7 +20,7 @@ func TestLeader_RegisterMember(t *testing.T) {
 	defer c1.Shutdown()
 
 	client := rpcClient(t, s1)
-	testutil.WaitForLeader(t, client.Call)
+	testutil.WaitForLeader(t, client.Call, "dc1")
 
 	// Try to join
 	addr := fmt.Sprintf("127.0.0.1:%d",
diff --git a/testutil/wait.go b/testutil/wait.go
index 20ab8ea5e5c1..7eb49a8e4d28 100644
--- a/testutil/wait.go
+++ b/testutil/wait.go
@@ -29,11 +29,11 @@ func WaitForResult(test testFn, error errorFn) {
 
 type rpcFn func(string, interface {}, interface {}) error
 
-func WaitForLeader(t *testing.T, rpc rpcFn) structs.IndexedNodes {
+func WaitForLeader(t *testing.T, rpc rpcFn, dc string) structs.IndexedNodes {
 	var out structs.IndexedNodes
 	WaitForResult(func() (bool, error) {
 		args := &structs.RegisterRequest{
-			Datacenter: "dc1",
+			Datacenter: dc,
 		}
 		err := rpc("Catalog.ListNodes", args, &out)
 		return out.QueryMeta.KnownLeader, err

From f202123fb1ba6e3a597097ecb1716688e731e71d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?William=20Tisa=CC=88ter?= <william@defunct.cc>
Date: Fri, 9 May 2014 01:18:08 +0200
Subject: [PATCH 17/39] Remove all sleeps from `health_endpoint_test.go`

---
 consul/health_endpoint_test.go | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/consul/health_endpoint_test.go b/consul/health_endpoint_test.go
index 465f3584671d..b3dc39f4311d 100644
--- a/consul/health_endpoint_test.go
+++ b/consul/health_endpoint_test.go
@@ -14,8 +14,7 @@ func TestHealth_ChecksInState(t *testing.T) {
 	client := rpcClient(t, s1)
 	defer client.Close()
 
-	// Wait for leader
-	time.Sleep(100 * time.Millisecond)
+	testutil.WaitForLeader(t, client.Call, "dc1")
 
 	arg := structs.RegisterRequest{
 		Datacenter: "dc1",
@@ -61,8 +60,7 @@ func TestHealth_NodeChecks(t *testing.T) {
 	client := rpcClient(t, s1)
 	defer client.Close()
 
-	// Wait for leader
-	time.Sleep(100 * time.Millisecond)
+	testutil.WaitForLeader(t, client.Call, "dc1")
 
 	arg := structs.RegisterRequest{
 		Datacenter: "dc1",
@@ -103,8 +101,7 @@ func TestHealth_ServiceChecks(t *testing.T) {
 	client := rpcClient(t, s1)
 	defer client.Close()
 
-	// Wait for leader
-	time.Sleep(100 * time.Millisecond)
+	testutil.WaitForLeader(t, client.Call, "dc1")
 
 	arg := structs.RegisterRequest{
 		Datacenter: "dc1",
@@ -150,8 +147,7 @@ func TestHealth_ServiceNodes(t *testing.T) {
 	client := rpcClient(t, s1)
 	defer client.Close()
 
-	// Wait for leader
-	time.Sleep(100 * time.Millisecond)
+	testutil.WaitForLeader(t, client.Call, "dc1")
 
 	arg := structs.RegisterRequest{
 		Datacenter: "dc1",

From 14dec397f8ba9173287efea3b7a14b7e7597d184 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?William=20Tisa=CC=88ter?= <william@defunct.cc>
Date: Fri, 9 May 2014 01:18:35 +0200
Subject: [PATCH 18/39] Remove all sleeps from `catalog_endpoint_test.go`

---
 consul/catalog_endpoint_test.go | 57 +++++++++++++--------------------
 1 file changed, 23 insertions(+), 34 deletions(-)

diff --git a/consul/catalog_endpoint_test.go b/consul/catalog_endpoint_test.go
index 149651de6b33..25a35dba08cc 100644
--- a/consul/catalog_endpoint_test.go
+++ b/consul/catalog_endpoint_test.go
@@ -64,8 +64,8 @@ func TestCatalogRegister_ForwardLeader(t *testing.T) {
 		t.Fatalf("err: %v", err)
 	}
 
-	// Wait for a leader
-	time.Sleep(100 * time.Millisecond)
+	testutil.WaitForLeader(t, client1.Call, "dc1")
+	testutil.WaitForLeader(t, client2.Call, "dc1")
 
 	// Use the follower as the client
 	var client *rpc.Client
@@ -109,8 +109,7 @@ func TestCatalogRegister_ForwardDC(t *testing.T) {
 		t.Fatalf("err: %v", err)
 	}
 
-	// Wait for the leaders
-	time.Sleep(100 * time.Millisecond)
+	testutil.WaitForLeader(t, client.Call, "dc2")
 
 	arg := structs.RegisterRequest{
 		Datacenter: "dc2", // SHould forward through s1
@@ -146,8 +145,7 @@ func TestCatalogDeregister(t *testing.T) {
 		t.Fatalf("err: %v", err)
 	}
 
-	// Wait for leader
-	time.Sleep(100 * time.Millisecond)
+	testutil.WaitForLeader(t, client.Call, "dc1")
 
 	if err := client.Call("Catalog.Deregister", &arg, &out); err != nil {
 		t.Fatalf("err: %v", err)
@@ -171,7 +169,8 @@ func TestCatalogListDatacenters(t *testing.T) {
 	if _, err := s2.JoinWAN([]string{addr}); err != nil {
 		t.Fatalf("err: %v", err)
 	}
-	time.Sleep(10 * time.Millisecond)
+
+	testutil.WaitForLeader(t, client.Call, "dc1")
 
 	var out []string
 	if err := client.Call("Catalog.ListDatacenters", struct{}{}, &out); err != nil {
@@ -208,8 +207,7 @@ func TestCatalogListNodes(t *testing.T) {
 		t.Fatalf("err: %v", err)
 	}
 
-	// Wait for leader
-	time.Sleep(100 * time.Millisecond)
+	testutil.WaitForLeader(t, client.Call, "dc1")
 
 	// Just add a node
 	s1.fsm.State().EnsureNode(1, structs.Node{"foo", "127.0.0.1"})
@@ -254,8 +252,8 @@ func TestCatalogListNodes_StaleRaad(t *testing.T) {
 		t.Fatalf("err: %v", err)
 	}
 
-	testutil.WaitForLeader(t, client1.Call)
-	testutil.WaitForLeader(t, client2.Call)
+	testutil.WaitForLeader(t, client1.Call, "dc1")
+	testutil.WaitForLeader(t, client2.Call, "dc1")
 
 	// Use the follower as the client
 	var client *rpc.Client
@@ -318,8 +316,8 @@ func TestCatalogListNodes_ConsistentRead_Fail(t *testing.T) {
 		t.Fatalf("err: %v", err)
 	}
 
-	// Wait for a leader
-	time.Sleep(100 * time.Millisecond)
+	testutil.WaitForLeader(t, client1.Call, "dc1")
+	testutil.WaitForLeader(t, client2.Call, "dc1")
 
 	// Use the leader as the client, kill the follower
 	var client *rpc.Client
@@ -368,8 +366,8 @@ func TestCatalogListNodes_ConsistentRead(t *testing.T) {
 		t.Fatalf("err: %v", err)
 	}
 
-	// Wait for a leader
-	time.Sleep(100 * time.Millisecond)
+	testutil.WaitForLeader(t, client1.Call, "dc1")
+	testutil.WaitForLeader(t, client2.Call, "dc1")
 
 	// Use the leader as the client, kill the follower
 	var client *rpc.Client
@@ -403,9 +401,6 @@ func BenchmarkCatalogListNodes(t *testing.B) {
 	client := rpcClient(nil, s1)
 	defer client.Close()
 
-	// Wait for leader
-	time.Sleep(100 * time.Millisecond)
-
 	// Just add a node
 	s1.fsm.State().EnsureNode(1, structs.Node{"foo", "127.0.0.1"})
 
@@ -436,8 +431,7 @@ func TestCatalogListServices(t *testing.T) {
 		t.Fatalf("err: %v", err)
 	}
 
-	// Wait for leader
-	time.Sleep(100 * time.Millisecond)
+	testutil.WaitForLeader(t, client.Call, "dc1")
 
 	// Just add a node
 	s1.fsm.State().EnsureNode(1, structs.Node{"foo", "127.0.0.1"})
@@ -474,8 +468,7 @@ func TestCatalogListServices_Blocking(t *testing.T) {
 	}
 	var out structs.IndexedServices
 
-	// Wait for leader
-	time.Sleep(100 * time.Millisecond)
+	testutil.WaitForLeader(t, client.Call, "dc1")
 
 	// Run the query
 	if err := client.Call("Catalog.ListServices", &args, &out); err != nil {
@@ -489,7 +482,7 @@ func TestCatalogListServices_Blocking(t *testing.T) {
 	// Async cause a change
 	start := time.Now()
 	go func() {
-		time.Sleep(100 * time.Millisecond)
+		time.Sleep(50 * time.Millisecond)
 		s1.fsm.State().EnsureNode(1, structs.Node{"foo", "127.0.0.1"})
 		s1.fsm.State().EnsureService(2, "foo", &structs.NodeService{"db", "db", []string{"primary"}, 5000})
 	}()
@@ -501,7 +494,7 @@ func TestCatalogListServices_Blocking(t *testing.T) {
 	}
 
 	// Should block at least 100ms
-	if time.Now().Sub(start) < 100*time.Millisecond {
+	if time.Now().Sub(start) < 50 * time.Millisecond {
 		t.Fatalf("too fast")
 	}
 
@@ -528,8 +521,7 @@ func TestCatalogListServices_Timeout(t *testing.T) {
 	}
 	var out structs.IndexedServices
 
-	// Wait for leader
-	time.Sleep(100 * time.Millisecond)
+	testutil.WaitForLeader(t, client.Call, "dc1")
 
 	// Run the query
 	if err := client.Call("Catalog.ListServices", &args, &out); err != nil {
@@ -538,7 +530,7 @@ func TestCatalogListServices_Timeout(t *testing.T) {
 
 	// Setup a blocking query
 	args.MinQueryIndex = out.Index
-	args.MaxQueryTime = 100 * time.Millisecond
+	args.MaxQueryTime = 50 * time.Millisecond
 
 	// Re-run the query
 	start := time.Now()
@@ -548,7 +540,7 @@ func TestCatalogListServices_Timeout(t *testing.T) {
 	}
 
 	// Should block at least 100ms
-	if time.Now().Sub(start) < 100*time.Millisecond {
+	if time.Now().Sub(start) < 50 * time.Millisecond {
 		t.Fatalf("too fast")
 	}
 
@@ -610,8 +602,7 @@ func TestCatalogListServiceNodes(t *testing.T) {
 		t.Fatalf("err: %v", err)
 	}
 
-	// Wait for leader
-	time.Sleep(100 * time.Millisecond)
+	testutil.WaitForLeader(t, client.Call, "dc1")
 
 	// Just add a node
 	s1.fsm.State().EnsureNode(1, structs.Node{"foo", "127.0.0.1"})
@@ -654,8 +645,7 @@ func TestCatalogNodeServices(t *testing.T) {
 		t.Fatalf("err: %v", err)
 	}
 
-	// Wait for leader
-	time.Sleep(100 * time.Millisecond)
+	testutil.WaitForLeader(t, client.Call, "dc1")
 
 	// Just add a node
 	s1.fsm.State().EnsureNode(1, structs.Node{"foo", "127.0.0.1"})
@@ -706,8 +696,7 @@ func TestCatalogRegister_FailedCase1(t *testing.T) {
 		t.Fatalf("err: %v", err)
 	}
 
-	// Wait for leader
-	time.Sleep(100 * time.Millisecond)
+	testutil.WaitForLeader(t, client.Call, "dc1")
 
 	if err := client.Call("Catalog.Register", &arg, &out); err != nil {
 		t.Fatalf("err: %v", err)

From fcb4f4ef95b750335dad0838fc89326d830179db Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?William=20Tisa=CC=88ter?= <william@defunct.cc>
Date: Fri, 9 May 2014 01:19:05 +0200
Subject: [PATCH 19/39] Remove most sleeps from `rpc_client_test.go`

---
 command/agent/rpc_client_test.go | 74 +++++++++++++-------------------
 1 file changed, 30 insertions(+), 44 deletions(-)

diff --git a/command/agent/rpc_client_test.go b/command/agent/rpc_client_test.go
index a31d6672a8a0..95b7292d8720 100644
--- a/command/agent/rpc_client_test.go
+++ b/command/agent/rpc_client_test.go
@@ -3,12 +3,13 @@ package agent
 import (
 	"fmt"
 	"github.com/hashicorp/serf/serf"
-	"github.com/hashicorp/serf/testutil"
+	"github.com/hashicorp/consul/testutil"
 	"io"
 	"net"
 	"os"
 	"strings"
 	"testing"
+	"errors"
 	"time"
 )
 
@@ -59,35 +60,32 @@ func TestRPCClientForceLeave(t *testing.T) {
 	p2 := testRPCClient(t)
 	defer p1.Close()
 	defer p2.Close()
-	testutil.Yield()
 
 	s2Addr := fmt.Sprintf("127.0.0.1:%d", p2.agent.config.Ports.SerfLan)
 	if _, err := p1.agent.JoinLAN([]string{s2Addr}); err != nil {
 		t.Fatalf("err: %s", err)
 	}
 
-	testutil.Yield()
-
 	if err := p2.agent.Shutdown(); err != nil {
 		t.Fatalf("err: %s", err)
 	}
 
-	time.Sleep(time.Second)
-
 	if err := p1.client.ForceLeave(p2.agent.config.NodeName); err != nil {
 		t.Fatalf("err: %s", err)
 	}
 
-	testutil.Yield()
-
 	m := p1.agent.LANMembers()
 	if len(m) != 2 {
 		t.Fatalf("should have 2 members: %#v", m)
 	}
 
-	if m[1].Status != serf.StatusLeft {
-		t.Fatalf("should be left: %#v %v", m[1], m[1].Status == serf.StatusLeft)
-	}
+	testutil.WaitForResult(func() (bool, error) {
+		m := p1.agent.LANMembers()
+		success := m[1].Status == serf.StatusLeft
+		return success, errors.New(m[1].Status.String())
+	}, func(err error) {
+		t.Fatalf("member status is %v, should be left", err)
+	})
 }
 
 func TestRPCClientJoinLAN(t *testing.T) {
@@ -95,7 +93,6 @@ func TestRPCClientJoinLAN(t *testing.T) {
 	p2 := testRPCClient(t)
 	defer p1.Close()
 	defer p2.Close()
-	testutil.Yield()
 
 	s2Addr := fmt.Sprintf("127.0.0.1:%d", p2.agent.config.Ports.SerfLan)
 	n, err := p1.client.Join([]string{s2Addr}, false)
@@ -113,7 +110,6 @@ func TestRPCClientJoinWAN(t *testing.T) {
 	p2 := testRPCClient(t)
 	defer p1.Close()
 	defer p2.Close()
-	testutil.Yield()
 
 	s2Addr := fmt.Sprintf("127.0.0.1:%d", p2.agent.config.Ports.SerfWan)
 	n, err := p1.client.Join([]string{s2Addr}, true)
@@ -131,7 +127,6 @@ func TestRPCClientLANMembers(t *testing.T) {
 	p2 := testRPCClient(t)
 	defer p1.Close()
 	defer p2.Close()
-	testutil.Yield()
 
 	mem, err := p1.client.LANMembers()
 	if err != nil {
@@ -148,8 +143,6 @@ func TestRPCClientLANMembers(t *testing.T) {
 		t.Fatalf("err: %s", err)
 	}
 
-	testutil.Yield()
-
 	mem, err = p1.client.LANMembers()
 	if err != nil {
 		t.Fatalf("err: %s", err)
@@ -165,7 +158,6 @@ func TestRPCClientWANMembers(t *testing.T) {
 	p2 := testRPCClient(t)
 	defer p1.Close()
 	defer p2.Close()
-	testutil.Yield()
 
 	mem, err := p1.client.WANMembers()
 	if err != nil {
@@ -182,8 +174,6 @@ func TestRPCClientWANMembers(t *testing.T) {
 		t.Fatalf("err: %s", err)
 	}
 
-	testutil.Yield()
-
 	mem, err = p1.client.WANMembers()
 	if err != nil {
 		t.Fatalf("err: %s", err)
@@ -194,16 +184,33 @@ func TestRPCClientWANMembers(t *testing.T) {
 	}
 }
 
+func TestRPCClientStats(t *testing.T) {
+	p1 := testRPCClient(t)
+	defer p1.Close()
+
+	stats, err := p1.client.Stats()
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	if _, ok := stats["agent"]; !ok {
+		t.Fatalf("bad: %#v", stats)
+	}
+
+	if _, ok := stats["consul"]; !ok {
+		t.Fatalf("bad: %#v", stats)
+	}
+}
+
 func TestRPCClientLeave(t *testing.T) {
 	p1 := testRPCClient(t)
 	defer p1.Close()
-	testutil.Yield()
 
 	if err := p1.client.Leave(); err != nil {
 		t.Fatalf("err: %s", err)
 	}
 
-	testutil.Yield()
+	time.Sleep(1 * time.Second)
 
 	select {
 	case <-p1.agent.ShutdownCh():
@@ -215,7 +222,6 @@ func TestRPCClientLeave(t *testing.T) {
 func TestRPCClientMonitor(t *testing.T) {
 	p1 := testRPCClient(t)
 	defer p1.Close()
-	testutil.Yield()
 
 	eventCh := make(chan string, 64)
 	if handle, err := p1.client.Monitor("debug", eventCh); err != nil {
@@ -224,8 +230,6 @@ func TestRPCClientMonitor(t *testing.T) {
 		defer p1.client.Stop(handle)
 	}
 
-	testutil.Yield()
-
 	found := false
 OUTER1:
 	for {
@@ -244,7 +248,8 @@ OUTER1:
 
 	// Join a bad thing to generate more events
 	p1.agent.JoinLAN(nil)
-	testutil.Yield()
+
+	time.Sleep(1 * time.Second)
 
 	found = false
 OUTER2:
@@ -262,22 +267,3 @@ OUTER2:
 		t.Fatalf("should log joining")
 	}
 }
-
-func TestRPCClientStats(t *testing.T) {
-	p1 := testRPCClient(t)
-	defer p1.Close()
-	testutil.Yield()
-
-	stats, err := p1.client.Stats()
-	if err != nil {
-		t.Fatalf("err: %s", err)
-	}
-
-	if _, ok := stats["agent"]; !ok {
-		t.Fatalf("bad: %#v", stats)
-	}
-
-	if _, ok := stats["consul"]; !ok {
-		t.Fatalf("bad: %#v", stats)
-	}
-}

From c0de985b91631dd732ef13df501da23c5fae76e1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?William=20Tisa=CC=88ter?= <william@defunct.cc>
Date: Fri, 9 May 2014 01:21:13 +0200
Subject: [PATCH 20/39] Fix imports in `health_endpoint_test.go`

---
 consul/health_endpoint_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/consul/health_endpoint_test.go b/consul/health_endpoint_test.go
index b3dc39f4311d..b817f9db0bdd 100644
--- a/consul/health_endpoint_test.go
+++ b/consul/health_endpoint_test.go
@@ -1,10 +1,10 @@
 package consul
 
 import (
+	"github.com/hashicorp/consul/testutil"
 	"github.com/hashicorp/consul/consul/structs"
 	"os"
 	"testing"
-	"time"
 )
 
 func TestHealth_ChecksInState(t *testing.T) {

From 176e3eb9dfa04634dac72e56a4af204f842ea3a2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?William=20Tisa=CC=88ter?= <william@defunct.cc>
Date: Fri, 9 May 2014 01:23:54 +0200
Subject: [PATCH 21/39] Remove all sleeps from `status_endpoint_test.go`

---
 consul/status_endpoint_test.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/consul/status_endpoint_test.go b/consul/status_endpoint_test.go
index 683b4bb74d01..75ff51947452 100644
--- a/consul/status_endpoint_test.go
+++ b/consul/status_endpoint_test.go
@@ -1,6 +1,7 @@
 package consul
 
 import (
+	"github.com/hashicorp/consul/testutil"
 	"github.com/ugorji/go/codec"
 	"net"
 	"net/rpc"
@@ -38,7 +39,7 @@ func TestStatusLeader(t *testing.T) {
 		t.Fatalf("unexpected leader: %v", leader)
 	}
 
-	time.Sleep(100 * time.Millisecond)
+	testutil.WaitForLeader(t, client.Call, "dc1")
 
 	if err := client.Call("Status.Leader", arg, &leader); err != nil {
 		t.Fatalf("err: %v", err)

From 4e9e14ff93232bc7ae9d7f3dfc0251a46737bd6d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?William=20Tisa=CC=88ter?= <william@defunct.cc>
Date: Fri, 9 May 2014 01:38:58 +0200
Subject: [PATCH 22/39] Remove all sleeps from `server_test.go`

---
 consul/server_test.go | 53 +++++++++++++++++++++++++------------------
 1 file changed, 31 insertions(+), 22 deletions(-)

diff --git a/consul/server_test.go b/consul/server_test.go
index 45edac53d97e..1b03cf83dee6 100644
--- a/consul/server_test.go
+++ b/consul/server_test.go
@@ -2,11 +2,13 @@ package consul
 
 import (
 	"fmt"
+	"github.com/hashicorp/consul/testutil"
 	"io/ioutil"
 	"net"
 	"os"
 	"testing"
 	"time"
+	"errors"
 )
 
 var nextPort = 15000
@@ -168,16 +170,16 @@ func TestServer_JoinWAN(t *testing.T) {
 		t.Fatalf("bad len")
 	}
 
-	time.Sleep(10 * time.Millisecond)
-
 	// Check the remoteConsuls has both
 	if len(s1.remoteConsuls) != 2 {
 		t.Fatalf("remote consul missing")
 	}
 
-	if len(s2.remoteConsuls) != 2 {
+	testutil.WaitForResult(func() (bool, error) {
+		return len(s2.remoteConsuls) == 2, nil
+	}, func(err error) {
 		t.Fatalf("remote consul missing")
-	}
+	})
 }
 
 func TestServer_Leave(t *testing.T) {
@@ -197,17 +199,22 @@ func TestServer_Leave(t *testing.T) {
 		t.Fatalf("err: %v", err)
 	}
 
-	time.Sleep(time.Second)
+	var p1 []net.Addr
+	var p2 []net.Addr
 
-	p1, _ := s1.raftPeers.Peers()
-	if len(p1) != 2 {
-		t.Fatalf("should have 2 peers: %v", p1)
-	}
+	testutil.WaitForResult(func() (bool, error) {
+		p1, _ = s1.raftPeers.Peers()
+		return len(p1) == 2, errors.New(fmt.Sprintf("%v", p1))
+	}, func(err error) {
+		t.Fatalf("should have 2 peers: %v", err)
+	})
 
-	p2, _ := s2.raftPeers.Peers()
-	if len(p2) != 2 {
-		t.Fatalf("should have 2 peers: %v", p2)
-	}
+	testutil.WaitForResult(func() (bool, error) {
+		p2, _ = s2.raftPeers.Peers()
+		return len(p2) == 2, errors.New(fmt.Sprintf("%v", p1))
+	}, func(err error) {
+		t.Fatalf("should have 2 peers: %v", err)
+	})
 
 	// Issue a leave
 	if err := s2.Leave(); err != nil {
@@ -272,14 +279,16 @@ func TestServer_JoinLAN_TLS(t *testing.T) {
 		t.Fatalf("bad len")
 	}
 
-	// Wait a while
-	time.Sleep(100 * time.Millisecond)
-
 	// Verify Raft has established a peer
-	if s1.Stats()["raft"]["num_peers"] != "1" {
-		t.Fatalf("bad: %v", s1.Stats()["raft"])
-	}
-	if s2.Stats()["raft"]["num_peers"] != "1" {
-		t.Fatalf("bad: %v", s2.Stats()["raft"])
-	}
+	testutil.WaitForResult(func() (bool, error) {
+		return s1.Stats()["raft"]["num_peers"] == "1", nil	
+	}, func(err error) {
+		t.Fatalf("no peer established")
+	})
+
+	testutil.WaitForResult(func() (bool, error) {
+		return s2.Stats()["raft"]["num_peers"] == "1", nil
+	}, func(err error) {
+		t.Fatalf("no peer established")
+	})
 }

From f4597403e3c1e5213bb18a54fb140ae89e72146b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?William=20Tisa=CC=88ter?= <william@defunct.cc>
Date: Fri, 9 May 2014 02:04:52 +0200
Subject: [PATCH 23/39] Remove all sleeps from `leader_test.go`

---
 consul/leader_test.go | 168 +++++++++++++++++-------------------------
 1 file changed, 66 insertions(+), 102 deletions(-)

diff --git a/consul/leader_test.go b/consul/leader_test.go
index f5b56174ebf9..955c00e225cd 100644
--- a/consul/leader_test.go
+++ b/consul/leader_test.go
@@ -7,7 +7,7 @@ import (
 	"github.com/hashicorp/serf/serf"
 	"os"
 	"testing"
-	"time"
+	"errors"
 )
 
 func TestLeader_RegisterMember(t *testing.T) {
@@ -19,9 +19,6 @@ func TestLeader_RegisterMember(t *testing.T) {
 	defer os.RemoveAll(dir2)
 	defer c1.Shutdown()
 
-	client := rpcClient(t, s1)
-	testutil.WaitForLeader(t, client.Call, "dc1")
-
 	// Try to join
 	addr := fmt.Sprintf("127.0.0.1:%d",
 		s1.config.SerfLANConfig.MemberlistConfig.BindPort)
@@ -29,8 +26,8 @@ func TestLeader_RegisterMember(t *testing.T) {
 		t.Fatalf("err: %v", err)
 	}
 
-	// Wait for registration
-	time.Sleep(10 * time.Millisecond)
+	client := rpcClient(t, s1)
+	testutil.WaitForLeader(t, client.Call, "dc1")
 
 	// Client should be registered
 	state := s1.fsm.State()
@@ -76,8 +73,8 @@ func TestLeader_FailedMember(t *testing.T) {
 	defer os.RemoveAll(dir2)
 	defer c1.Shutdown()
 
-	// Wait until we have a leader
-	time.Sleep(100 * time.Millisecond)
+	client := rpcClient(t, s1)
+	testutil.WaitForLeader(t, client.Call, "dc1")
 
 	// Try to join
 	addr := fmt.Sprintf("127.0.0.1:%d",
@@ -89,9 +86,6 @@ func TestLeader_FailedMember(t *testing.T) {
 	// Fail the member
 	c1.Shutdown()
 
-	// Wait for failure detection
-	time.Sleep(500 * time.Millisecond)
-
 	// Should be registered
 	state := s1.fsm.State()
 	_, found, _ := state.GetNode(c1.config.NodeName)
@@ -110,9 +104,13 @@ func TestLeader_FailedMember(t *testing.T) {
 	if checks[0].Name != SerfCheckName {
 		t.Fatalf("bad check: %v", checks[0])
 	}
-	if checks[0].Status != structs.HealthCritical {
-		t.Fatalf("bad check: %v", checks[0])
-	}
+
+	testutil.WaitForResult(func() (bool, error) {
+		_, checks = state.NodeChecks(c1.config.NodeName)
+		return checks[0].Status == structs.HealthCritical, errors.New(checks[0].Status)
+	}, func(err error) {
+		t.Fatalf("check status is %v, should be critical", err)
+	})
 }
 
 func TestLeader_LeftMember(t *testing.T) {
@@ -124,9 +122,6 @@ func TestLeader_LeftMember(t *testing.T) {
 	defer os.RemoveAll(dir2)
 	defer c1.Shutdown()
 
-	// Wait until we have a leader
-	time.Sleep(100 * time.Millisecond)
-
 	// Try to join
 	addr := fmt.Sprintf("127.0.0.1:%d",
 		s1.config.SerfLANConfig.MemberlistConfig.BindPort)
@@ -134,28 +129,28 @@ func TestLeader_LeftMember(t *testing.T) {
 		t.Fatalf("err: %v", err)
 	}
 
-	// Wait for registration
-	time.Sleep(10 * time.Millisecond)
+	var found bool
+	state := s1.fsm.State()
 
 	// Should be registered
-	state := s1.fsm.State()
-	_, found, _ := state.GetNode(c1.config.NodeName)
-	if !found {
-		t.Fatalf("client not registered")
-	}
+	testutil.WaitForResult(func() (bool, error) {
+		_, found, _ = state.GetNode(c1.config.NodeName)
+		return found == true, nil
+	}, func(err error) {
+		t.Fatalf("client should be registered")
+	})
 
 	// Node should leave
 	c1.Leave()
 	c1.Shutdown()
 
-	// Wait for failure detection
-	time.Sleep(500 * time.Millisecond)
-
 	// Should be deregistered
-	_, found, _ = state.GetNode(c1.config.NodeName)
-	if found {
-		t.Fatalf("client registered")
-	}
+	testutil.WaitForResult(func() (bool, error) {
+		_, found, _ = state.GetNode(c1.config.NodeName)
+		return found == false, nil
+	}, func(err error) {
+		t.Fatalf("client should not be registered")
+	})
 }
 
 func TestLeader_ReapMember(t *testing.T) {
@@ -167,9 +162,6 @@ func TestLeader_ReapMember(t *testing.T) {
 	defer os.RemoveAll(dir2)
 	defer c1.Shutdown()
 
-	// Wait until we have a leader
-	time.Sleep(100 * time.Millisecond)
-
 	// Try to join
 	addr := fmt.Sprintf("127.0.0.1:%d",
 		s1.config.SerfLANConfig.MemberlistConfig.BindPort)
@@ -177,15 +169,16 @@ func TestLeader_ReapMember(t *testing.T) {
 		t.Fatalf("err: %v", err)
 	}
 
-	// Wait for registration
-	time.Sleep(10 * time.Millisecond)
+	var found bool
+	state := s1.fsm.State()
 
 	// Should be registered
-	state := s1.fsm.State()
-	_, found, _ := state.GetNode(c1.config.NodeName)
-	if !found {
-		t.Fatalf("client not registered")
-	}
+	testutil.WaitForResult(func() (bool, error) {
+		_, found, _ = state.GetNode(c1.config.NodeName)
+		return found == true, nil
+	}, func(err error) {
+		t.Fatalf("client should be registered")
+	})
 
 	// Simulate a node reaping
 	mems := s1.LANMembers()
@@ -199,14 +192,13 @@ func TestLeader_ReapMember(t *testing.T) {
 	}
 	s1.reconcileCh <- c1mem
 
-	// Wait to reconcile
-	time.Sleep(10 * time.Millisecond)
-
 	// Should be deregistered
-	_, found, _ = state.GetNode(c1.config.NodeName)
-	if found {
-		t.Fatalf("client registered")
-	}
+	testutil.WaitForResult(func() (bool, error) {
+		_, found, _ = state.GetNode(c1.config.NodeName)
+		return found == false, nil
+	}, func(err error) {
+		t.Fatalf("client should not be registered")
+	})
 }
 
 func TestLeader_Reconcile_ReapMember(t *testing.T) {
@@ -214,8 +206,8 @@ func TestLeader_Reconcile_ReapMember(t *testing.T) {
 	defer os.RemoveAll(dir1)
 	defer s1.Shutdown()
 
-	// Wait until we have a leader
-	time.Sleep(100 * time.Millisecond)
+	client := rpcClient(t, s1)
+	testutil.WaitForLeader(t, client.Call, "dc1")
 
 	// Register a non-existing member
 	dead := structs.RegisterRequest{
@@ -270,14 +262,13 @@ func TestLeader_Reconcile(t *testing.T) {
 		t.Fatalf("client registered")
 	}
 
-	// Wait for leader
-	time.Sleep(100 * time.Millisecond)
-
 	// Should be registered
-	_, found, _ = state.GetNode(c1.config.NodeName)
-	if !found {
-		t.Fatalf("client not registered")
-	}
+	testutil.WaitForResult(func() (bool, error) {
+		_, found, _ = state.GetNode(c1.config.NodeName)
+		return found == true, nil
+	}, func(err error) {
+		t.Fatalf("client should be registered")
+	})
 }
 
 func TestLeader_LeftServer(t *testing.T) {
@@ -304,48 +295,30 @@ func TestLeader_LeftServer(t *testing.T) {
 		t.Fatalf("err: %v", err)
 	}
 
-	// Wait until we have 3 peers
-	start := time.Now()
-CHECK1:
 	for _, s := range servers {
-		peers, _ := s.raftPeers.Peers()
-		if len(peers) != 3 {
-			if time.Now().Sub(start) >= 2*time.Second {
-				t.Fatalf("should have 3 peers")
-			} else {
-				time.Sleep(100 * time.Millisecond)
-				goto CHECK1
-			}
-		}
+		testutil.WaitForResult(func() (bool, error) {
+			peers, _ := s.raftPeers.Peers()
+			return len(peers) == 3, nil
+		}, func(err error) {
+			t.Fatalf("should have 3 peers")
+		})
 	}
 
 	// Kill any server
 	servers[0].Shutdown()
 
-	// Wait for failure detection
-	time.Sleep(500 * time.Millisecond)
-
 	// Force remove the non-leader (transition to left state)
 	if err := servers[1].RemoveFailedNode(servers[0].config.NodeName); err != nil {
 		t.Fatalf("err: %v", err)
 	}
 
-	// Wait for intent propagation
-	time.Sleep(500 * time.Millisecond)
-
-	// Wait until we have 2 peers
-	start = time.Now()
-CHECK2:
 	for _, s := range servers[1:] {
-		peers, _ := s.raftPeers.Peers()
-		if len(peers) != 2 {
-			if time.Now().Sub(start) >= 2*time.Second {
-				t.Fatalf("should have 2 peers")
-			} else {
-				time.Sleep(100 * time.Millisecond)
-				goto CHECK2
-			}
-		}
+		testutil.WaitForResult(func() (bool, error) {
+			peers, _ := s.raftPeers.Peers()
+			return len(peers) == 2, nil
+		}, func(err error) {
+			t.Fatalf("should have 2 peers")
+		})
 	}
 }
 
@@ -367,24 +340,15 @@ func TestLeader_MultiBootstrap(t *testing.T) {
 		t.Fatalf("err: %v", err)
 	}
 
-	// Wait until we have 2 peers
-	start := time.Now()
-CHECK1:
 	for _, s := range servers {
-		peers := s.serfLAN.Members()
-		if len(peers) != 2 {
-			if time.Now().Sub(start) >= 2*time.Second {
-				t.Fatalf("should have 2 peers")
-			} else {
-				time.Sleep(100 * time.Millisecond)
-				goto CHECK1
-			}
-		}
+		testutil.WaitForResult(func() (bool, error) {
+			peers := s.serfLAN.Members()
+			return len(peers) == 2, nil
+		}, func(err error) {
+			t.Fatalf("should have 2 peers")
+		})
 	}
 
-	// Wait to ensure no peer is added
-	time.Sleep(200 * time.Millisecond)
-
 	// Ensure we don't have multiple raft peers
 	for _, s := range servers {
 		peers, _ := s.raftPeers.Peers()

From 69eef2b4551fca263818cf595f78e6f1fbadc6b2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?William=20Tisa=CC=88ter?= <william@defunct.cc>
Date: Fri, 9 May 2014 02:06:19 +0200
Subject: [PATCH 24/39] Remove all sleeps from `status_endpoint_test.go

---
 command/agent/status_endpoint_test.go | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/command/agent/status_endpoint_test.go b/command/agent/status_endpoint_test.go
index 835b80bd9f26..4dd1f4e204a9 100644
--- a/command/agent/status_endpoint_test.go
+++ b/command/agent/status_endpoint_test.go
@@ -3,7 +3,7 @@ package agent
 import (
 	"os"
 	"testing"
-	"time"
+	"github.com/hashicorp/consul/testutil"
 )
 
 func TestStatusLeader(t *testing.T) {
@@ -12,8 +12,7 @@ func TestStatusLeader(t *testing.T) {
 	defer srv.Shutdown()
 	defer srv.agent.Shutdown()
 
-	// Wait for a leader
-	time.Sleep(100 * time.Millisecond)
+	testutil.WaitForLeader(t, srv.agent.RPC, "dc1")
 
 	obj, err := srv.StatusLeader(nil, nil)
 	if err != nil {

From 2d36b0f0177746e4b16001b07e7e77b09ca66b6c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?William=20Tisa=CC=88ter?= <william@defunct.cc>
Date: Fri, 9 May 2014 02:08:01 +0200
Subject: [PATCH 25/39] Remove all sleeps from `ui_endpoint_test.go`

---
 command/agent/ui_endpoint_test.go | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/command/agent/ui_endpoint_test.go b/command/agent/ui_endpoint_test.go
index c056e433ee36..ab734f3cb471 100644
--- a/command/agent/ui_endpoint_test.go
+++ b/command/agent/ui_endpoint_test.go
@@ -3,6 +3,7 @@ package agent
 import (
 	"bytes"
 	"fmt"
+	"github.com/hashicorp/consul/testutil"
 	"github.com/hashicorp/consul/consul/structs"
 	"io"
 	"io/ioutil"
@@ -12,7 +13,6 @@ import (
 	"path/filepath"
 	"reflect"
 	"testing"
-	"time"
 )
 
 func TestUiIndex(t *testing.T) {
@@ -60,8 +60,7 @@ func TestUiNodes(t *testing.T) {
 	defer srv.Shutdown()
 	defer srv.agent.Shutdown()
 
-	// Wait for leader
-	time.Sleep(100 * time.Millisecond)
+	testutil.WaitForLeader(t, srv.agent.RPC, "dc1")
 
 	req, err := http.NewRequest("GET", "/v1/internal/ui/nodes/dc1", nil)
 	if err != nil {
@@ -88,8 +87,7 @@ func TestUiNodeInfo(t *testing.T) {
 	defer srv.Shutdown()
 	defer srv.agent.Shutdown()
 
-	// Wait for leader
-	time.Sleep(100 * time.Millisecond)
+	testutil.WaitForLeader(t, srv.agent.RPC, "dc1")
 
 	req, err := http.NewRequest("GET",
 		fmt.Sprintf("/v1/internal/ui/node/%s", srv.agent.config.NodeName), nil)

From 9e73d1b8499790e15f6a2a6b4d9165ec46eb947f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?William=20Tisa=CC=88ter?= <william@defunct.cc>
Date: Fri, 9 May 2014 02:09:53 +0200
Subject: [PATCH 26/39] Remove all sleeps from `kvs_endpoint_test.go`

---
 consul/kvs_endpoint_test.go | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/consul/kvs_endpoint_test.go b/consul/kvs_endpoint_test.go
index 5c3eef621124..32ab830ea998 100644
--- a/consul/kvs_endpoint_test.go
+++ b/consul/kvs_endpoint_test.go
@@ -1,10 +1,10 @@
 package consul
 
 import (
+	"github.com/hashicorp/consul/testutil"
 	"github.com/hashicorp/consul/consul/structs"
 	"os"
 	"testing"
-	"time"
 )
 
 func TestKVS_Apply(t *testing.T) {
@@ -14,8 +14,7 @@ func TestKVS_Apply(t *testing.T) {
 	client := rpcClient(t, s1)
 	defer client.Close()
 
-	// Wait for leader
-	time.Sleep(100 * time.Millisecond)
+	testutil.WaitForLeader(t, client.Call, "dc1")
 
 	arg := structs.KVSRequest{
 		Datacenter: "dc1",
@@ -71,8 +70,7 @@ func TestKVS_Get(t *testing.T) {
 	client := rpcClient(t, s1)
 	defer client.Close()
 
-	// Wait for leader
-	time.Sleep(100 * time.Millisecond)
+	testutil.WaitForLeader(t, client.Call, "dc1")
 
 	arg := structs.KVSRequest{
 		Datacenter: "dc1",
@@ -119,8 +117,7 @@ func TestKVSEndpoint_List(t *testing.T) {
 	client := rpcClient(t, s1)
 	defer client.Close()
 
-	// Wait for leader
-	time.Sleep(100 * time.Millisecond)
+	testutil.WaitForLeader(t, client.Call, "dc1")
 
 	keys := []string{
 		"/test/key1",
@@ -179,8 +176,7 @@ func TestKVSEndpoint_ListKeys(t *testing.T) {
 	client := rpcClient(t, s1)
 	defer client.Close()
 
-	// Wait for leader
-	time.Sleep(100 * time.Millisecond)
+	testutil.WaitForLeader(t, client.Call, "dc1")
 
 	keys := []string{
 		"/test/key1",

From f0a8866d42728f584e5c8c6b47b140b8ab681808 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?William=20Tisa=CC=88ter?= <william@defunct.cc>
Date: Fri, 9 May 2014 02:11:40 +0200
Subject: [PATCH 27/39] Remove all sleeps from `internal_endpoint_test.go`

---
 consul/internal_endpoint_test.go | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/consul/internal_endpoint_test.go b/consul/internal_endpoint_test.go
index a5ff17373ca7..9b08fe74da8f 100644
--- a/consul/internal_endpoint_test.go
+++ b/consul/internal_endpoint_test.go
@@ -1,10 +1,10 @@
 package consul
 
 import (
+	"github.com/hashicorp/consul/testutil"
 	"github.com/hashicorp/consul/consul/structs"
 	"os"
 	"testing"
-	"time"
 )
 
 func TestInternal_NodeInfo(t *testing.T) {
@@ -14,8 +14,7 @@ func TestInternal_NodeInfo(t *testing.T) {
 	client := rpcClient(t, s1)
 	defer client.Close()
 
-	// Wait for leader
-	time.Sleep(100 * time.Millisecond)
+	testutil.WaitForLeader(t, client.Call, "dc1")
 
 	arg := structs.RegisterRequest{
 		Datacenter: "dc1",
@@ -68,8 +67,7 @@ func TestInternal_NodeDump(t *testing.T) {
 	client := rpcClient(t, s1)
 	defer client.Close()
 
-	// Wait for leader
-	time.Sleep(100 * time.Millisecond)
+	testutil.WaitForLeader(t, client.Call, "dc1")
 
 	arg := structs.RegisterRequest{
 		Datacenter: "dc1",

From 7ffc8cd29e79d0c42ec7839cb0678a251809a5d6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?William=20Tisa=CC=88ter?= <william@defunct.cc>
Date: Fri, 9 May 2014 02:14:39 +0200
Subject: [PATCH 28/39] Fix race condition in `TestLeader_FailedMember`

---
 consul/leader_test.go | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/consul/leader_test.go b/consul/leader_test.go
index 955c00e225cd..39935c2aed3f 100644
--- a/consul/leader_test.go
+++ b/consul/leader_test.go
@@ -88,10 +88,12 @@ func TestLeader_FailedMember(t *testing.T) {
 
 	// Should be registered
 	state := s1.fsm.State()
-	_, found, _ := state.GetNode(c1.config.NodeName)
-	if !found {
+	testutil.WaitForResult(func() (bool, error) {
+		_, found, _ := state.GetNode(c1.config.NodeName)
+		return found == true, nil
+	}, func(err error) {
 		t.Fatalf("client not registered")
-	}
+	})
 
 	// Should have a check
 	_, checks := state.NodeChecks(c1.config.NodeName)

From a727c59e74ca1908e100b65b6a2dfb549cec252e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?William=20Tisa=CC=88ter?= <william@defunct.cc>
Date: Fri, 9 May 2014 02:29:51 +0200
Subject: [PATCH 29/39] Remove all sleeps from `client_test.go`

---
 consul/client_test.go | 31 +++++++++++++++++--------------
 1 file changed, 17 insertions(+), 14 deletions(-)

diff --git a/consul/client_test.go b/consul/client_test.go
index 7aff15fcefd1..b0b748eef102 100644
--- a/consul/client_test.go
+++ b/consul/client_test.go
@@ -2,6 +2,7 @@ package consul
 
 import (
 	"fmt"
+	"github.com/hashicorp/consul/testutil"
 	"github.com/hashicorp/consul/consul/structs"
 	"net"
 	"os"
@@ -80,12 +81,12 @@ func TestClient_JoinLAN(t *testing.T) {
 		t.Fatalf("bad len")
 	}
 
-	time.Sleep(10 * time.Millisecond)
-
 	// Check we have a new consul
-	if len(c1.consuls) != 1 {
+	testutil.WaitForResult(func() (bool, error) {
+		return len(c1.consuls) == 1, nil
+	}, func(err error) {
 		t.Fatalf("expected consul server")
-	}
+	})
 }
 
 func TestClient_RPC(t *testing.T) {
@@ -119,12 +120,13 @@ func TestClient_RPC(t *testing.T) {
 		t.Fatalf("bad len")
 	}
 
-	time.Sleep(10 * time.Millisecond)
-
-	// RPC shoudl succeed
-	if err := c1.RPC("Status.Ping", struct{}{}, &out); err != nil {
+	// RPC should succeed
+	testutil.WaitForResult(func() (bool, error) {
+		err := c1.RPC("Status.Ping", struct{}{}, &out)
+		return err == nil, err
+	}, func(err error) {
 		t.Fatalf("err: %v", err)
-	}
+	})
 }
 
 func TestClient_RPC_TLS(t *testing.T) {
@@ -171,10 +173,11 @@ func TestClient_RPC_TLS(t *testing.T) {
 		t.Fatalf("bad len")
 	}
 
-	time.Sleep(10 * time.Millisecond)
-
-	// RPC shoudl succeed
-	if err := c1.RPC("Status.Ping", struct{}{}, &out); err != nil {
+	// RPC should succeed
+	testutil.WaitForResult(func() (bool, error) {
+		err := c1.RPC("Status.Ping", struct{}{}, &out)
+		return err == nil, err
+	}, func(err error) {
 		t.Fatalf("err: %v", err)
-	}
+	})
 }

From 8e6ae4019ba2cd6e2a9370a8c8c2eccba6bd57a9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?William=20Tisa=CC=88ter?= <william@defunct.cc>
Date: Fri, 9 May 2014 02:30:05 +0200
Subject: [PATCH 30/39] Fix race condition in `TestHTTPAgentJoin_WAN`

---
 command/agent/agent_endpoint_test.go | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/command/agent/agent_endpoint_test.go b/command/agent/agent_endpoint_test.go
index 78406de731a5..fee1b28a624f 100644
--- a/command/agent/agent_endpoint_test.go
+++ b/command/agent/agent_endpoint_test.go
@@ -169,9 +169,11 @@ func TestHTTPAgentJoin_WAN(t *testing.T) {
 		t.Fatalf("Err: %v", obj)
 	}
 
-	if len(a2.WANMembers()) != 2 {
+	testutil.WaitForResult(func() (bool, error) {
+		return len(a2.WANMembers()) == 2, nil
+	}, func(err error) {
 		t.Fatalf("should have 2 members")
-	}
+	})
 }
 
 func TestHTTPAgentForceLeave(t *testing.T) {

From 860cdb9c968611dc6e0ff8ae1e3ae70cdcc2b22d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?William=20Tisa=CC=88ter?= <william@defunct.cc>
Date: Fri, 9 May 2014 02:36:25 +0200
Subject: [PATCH 31/39] Change blocking call sleep time

---
 consul/catalog_endpoint_test.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/consul/catalog_endpoint_test.go b/consul/catalog_endpoint_test.go
index 25a35dba08cc..3db8be3f2905 100644
--- a/consul/catalog_endpoint_test.go
+++ b/consul/catalog_endpoint_test.go
@@ -482,7 +482,7 @@ func TestCatalogListServices_Blocking(t *testing.T) {
 	// Async cause a change
 	start := time.Now()
 	go func() {
-		time.Sleep(50 * time.Millisecond)
+		time.Sleep(100 * time.Millisecond)
 		s1.fsm.State().EnsureNode(1, structs.Node{"foo", "127.0.0.1"})
 		s1.fsm.State().EnsureService(2, "foo", &structs.NodeService{"db", "db", []string{"primary"}, 5000})
 	}()
@@ -494,7 +494,7 @@ func TestCatalogListServices_Blocking(t *testing.T) {
 	}
 
 	// Should block at least 100ms
-	if time.Now().Sub(start) < 50 * time.Millisecond {
+	if time.Now().Sub(start) < 100 * time.Millisecond {
 		t.Fatalf("too fast")
 	}
 
@@ -530,7 +530,7 @@ func TestCatalogListServices_Timeout(t *testing.T) {
 
 	// Setup a blocking query
 	args.MinQueryIndex = out.Index
-	args.MaxQueryTime = 50 * time.Millisecond
+	args.MaxQueryTime = 100 * time.Millisecond
 
 	// Re-run the query
 	start := time.Now()
@@ -540,7 +540,7 @@ func TestCatalogListServices_Timeout(t *testing.T) {
 	}
 
 	// Should block at least 100ms
-	if time.Now().Sub(start) < 50 * time.Millisecond {
+	if time.Now().Sub(start) < 100 * time.Millisecond {
 		t.Fatalf("too fast")
 	}
 

From e1d111a43dd0f9b88358d3509ae8944a092733e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?William=20Tisa=CC=88ter?= <william@defunct.cc>
Date: Fri, 9 May 2014 02:43:48 +0200
Subject: [PATCH 32/39] Increase timeout for `WaitForResult`

---
 testutil/wait.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/testutil/wait.go b/testutil/wait.go
index 7eb49a8e4d28..dd51d8c26eba 100644
--- a/testutil/wait.go
+++ b/testutil/wait.go
@@ -10,10 +10,10 @@ type testFn func() (bool, error)
 type errorFn func(error)
 
 func WaitForResult(test testFn, error errorFn) {
-	retries := 100  // 5 seconds timeout
+	retries := 100
 
 	for retries > 0 {
-		time.Sleep(50 * time.Millisecond)
+		time.Sleep(100 * time.Millisecond)
 		retries--
 
 		success, err := test()

From 76d9fa255be7f727077bf48e2cc3acfb475f9975 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?William=20Tisa=CC=88ter?= <william@defunct.cc>
Date: Fri, 9 May 2014 03:00:43 +0200
Subject: [PATCH 33/39] Speed up test runs in `WaitForResult`

---
 testutil/wait.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/testutil/wait.go b/testutil/wait.go
index dd51d8c26eba..e3e0e9149b1a 100644
--- a/testutil/wait.go
+++ b/testutil/wait.go
@@ -10,10 +10,10 @@ type testFn func() (bool, error)
 type errorFn func(error)
 
 func WaitForResult(test testFn, error errorFn) {
-	retries := 100
+	retries := 1000
 
 	for retries > 0 {
-		time.Sleep(100 * time.Millisecond)
+		time.Sleep(10 * time.Millisecond)
 		retries--
 
 		success, err := test()

From 048b780d6171f0d7bb4536e074753a8dd559736e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?William=20Tisa=CC=88ter?= <william@defunct.cc>
Date: Fri, 9 May 2014 03:00:56 +0200
Subject: [PATCH 34/39] Fix race condition in `TestCatalogListNodes`

---
 consul/catalog_endpoint_test.go | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/consul/catalog_endpoint_test.go b/consul/catalog_endpoint_test.go
index 3db8be3f2905..3945dd51d92f 100644
--- a/consul/catalog_endpoint_test.go
+++ b/consul/catalog_endpoint_test.go
@@ -212,13 +212,12 @@ func TestCatalogListNodes(t *testing.T) {
 	// Just add a node
 	s1.fsm.State().EnsureNode(1, structs.Node{"foo", "127.0.0.1"})
 
-	if err := client.Call("Catalog.ListNodes", &args, &out); err != nil {
+	testutil.WaitForResult(func() (bool, error) {
+		client.Call("Catalog.ListNodes", &args, &out)
+		return len(out.Nodes) == 2, nil
+	}, func(err error) {
 		t.Fatalf("err: %v", err)
-	}
-
-	if len(out.Nodes) != 2 {
-		t.Fatalf("bad: %v", out)
-	}
+	})
 
 	// Server node is auto added from Serf
 	if out.Nodes[0].Node != s1.config.NodeName {

From 7092ab744b951a5dd00256934f7413773bd1d315 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?William=20Tisa=CC=88ter?= <william@defunct.cc>
Date: Fri, 9 May 2014 03:05:38 +0200
Subject: [PATCH 35/39] Fix race condition in `TestLeader_RegisterMember`

---
 consul/leader_test.go | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/consul/leader_test.go b/consul/leader_test.go
index 39935c2aed3f..b99924eebf20 100644
--- a/consul/leader_test.go
+++ b/consul/leader_test.go
@@ -31,10 +31,12 @@ func TestLeader_RegisterMember(t *testing.T) {
 
 	// Client should be registered
 	state := s1.fsm.State()
-	_, found, _ := state.GetNode(c1.config.NodeName)
-	if !found {
+	testutil.WaitForResult(func() (bool, error) {
+		_, found, _ := state.GetNode(c1.config.NodeName)
+		return found == true, nil
+	}, func(err error) {
 		t.Fatalf("client not registered")
-	}
+	})
 
 	// Should have a check
 	_, checks := state.NodeChecks(c1.config.NodeName)
@@ -52,7 +54,7 @@ func TestLeader_RegisterMember(t *testing.T) {
 	}
 
 	// Server should be registered
-	_, found, _ = state.GetNode(s1.config.NodeName)
+	_, found, _ := state.GetNode(s1.config.NodeName)
 	if !found {
 		t.Fatalf("server not registered")
 	}

From 4a9058a07eeb8c55ad83e45b712723156c7f2bc7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?William=20Tisa=CC=88ter?= <william@defunct.cc>
Date: Fri, 9 May 2014 03:24:13 +0200
Subject: [PATCH 36/39] Sleep since no obvious way to wait for shutdown

---
 consul/leader_test.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/consul/leader_test.go b/consul/leader_test.go
index b99924eebf20..8e4c4a63b101 100644
--- a/consul/leader_test.go
+++ b/consul/leader_test.go
@@ -8,6 +8,7 @@ import (
 	"os"
 	"testing"
 	"errors"
+	"time"
 )
 
 func TestLeader_RegisterMember(t *testing.T) {
@@ -310,6 +311,7 @@ func TestLeader_LeftServer(t *testing.T) {
 
 	// Kill any server
 	servers[0].Shutdown()
+	time.Sleep(100 * time.Millisecond)
 
 	// Force remove the non-leader (transition to left state)
 	if err := servers[1].RemoveFailedNode(servers[0].config.NodeName); err != nil {

From 0e865daf4c8ca8b90625dff5b871e761497dbee5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?William=20Tisa=CC=88ter?= <william@defunct.cc>
Date: Fri, 9 May 2014 03:41:10 +0200
Subject: [PATCH 37/39] Run checks more often in `expectStatus`

---
 command/agent/check_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/command/agent/check_test.go b/command/agent/check_test.go
index 54a9b15c9cd4..07b4f598d7e7 100644
--- a/command/agent/check_test.go
+++ b/command/agent/check_test.go
@@ -31,7 +31,7 @@ func expectStatus(t *testing.T, script, status string) {
 		Notify:   mock,
 		CheckID:  "foo",
 		Script:   script,
-		Interval: 25 * time.Millisecond,
+		Interval: 10 * time.Millisecond,
 		Logger:   log.New(os.Stderr, "", log.LstdFlags),
 	}
 	check.Start()

From 335cd5a68a3c1016e72e424deeb9e082e0f79156 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?William=20Tisa=CC=88ter?= <william@defunct.cc>
Date: Fri, 9 May 2014 03:41:47 +0200
Subject: [PATCH 38/39] Wait wrap tests in `TestServer_JoinLAN`

---
 consul/server_test.go | 36 ++++++++++++++++++++++++------------
 1 file changed, 24 insertions(+), 12 deletions(-)

diff --git a/consul/server_test.go b/consul/server_test.go
index 1b03cf83dee6..64342f0e5d00 100644
--- a/consul/server_test.go
+++ b/consul/server_test.go
@@ -136,13 +136,17 @@ func TestServer_JoinLAN(t *testing.T) {
 	}
 
 	// Check the members
-	if len(s1.LANMembers()) != 2 {
+	testutil.WaitForResult(func() (bool, error) {
+		return len(s1.LANMembers()) == 2, nil
+	}, func(err error) {
 		t.Fatalf("bad len")
-	}
+	})
 
-	if len(s2.LANMembers()) != 2 {
+	testutil.WaitForResult(func() (bool, error) {
+		return len(s2.LANMembers()) == 2, nil
+	}, func(err error) {
 		t.Fatalf("bad len")
-	}
+	})
 }
 
 func TestServer_JoinWAN(t *testing.T) {
@@ -162,13 +166,17 @@ func TestServer_JoinWAN(t *testing.T) {
 	}
 
 	// Check the members
-	if len(s1.WANMembers()) != 2 {
+	testutil.WaitForResult(func() (bool, error) {
+		return len(s1.WANMembers()) == 2, nil
+	}, func(err error) {
 		t.Fatalf("bad len")
-	}
+	})
 
-	if len(s2.WANMembers()) != 2 {
+	testutil.WaitForResult(func() (bool, error) {
+		return len(s2.WANMembers()) == 2, nil
+	}, func(err error) {
 		t.Fatalf("bad len")
-	}
+	})
 
 	// Check the remoteConsuls has both
 	if len(s1.remoteConsuls) != 2 {
@@ -271,13 +279,17 @@ func TestServer_JoinLAN_TLS(t *testing.T) {
 	}
 
 	// Check the members
-	if len(s1.LANMembers()) != 2 {
+	testutil.WaitForResult(func() (bool, error) {
+		return len(s1.LANMembers()) == 2, nil
+	}, func(err error) {
 		t.Fatalf("bad len")
-	}
+	})
 
-	if len(s2.LANMembers()) != 2 {
+	testutil.WaitForResult(func() (bool, error) {
+		return len(s2.LANMembers()) == 2, nil
+	}, func(err error) {
 		t.Fatalf("bad len")
-	}
+	})
 
 	// Verify Raft has established a peer
 	testutil.WaitForResult(func() (bool, error) {

From dcfd03be00c86190c14d5a61b2275aba9e18b327 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?William=20Tisa=CC=88ter?= <william@defunct.cc>
Date: Fri, 9 May 2014 11:38:29 +0200
Subject: [PATCH 39/39] Add TODOs for failing tests

---
 command/agent/catalog_endpoint_test.go | 1 +
 command/agent/health_endpoint_test.go  | 7 +++++++
 command/agent/rpc_client_test.go       | 1 +
 command/agent/ui_endpoint_test.go      | 2 ++
 consul/catalog_endpoint_test.go        | 1 +
 5 files changed, 12 insertions(+)

diff --git a/command/agent/catalog_endpoint_test.go b/command/agent/catalog_endpoint_test.go
index d9454ab7ed87..f8b2b6a2c701 100644
--- a/command/agent/catalog_endpoint_test.go
+++ b/command/agent/catalog_endpoint_test.go
@@ -175,6 +175,7 @@ func TestCatalogNodes_Blocking(t *testing.T) {
 
 	// Should block for a while
 	if time.Now().Sub(start) < 50 * time.Millisecond {
+		// TODO: Failing
 		t.Fatalf("too fast")
 	}
 
diff --git a/command/agent/health_endpoint_test.go b/command/agent/health_endpoint_test.go
index 910d37df8b10..a4b4b9817e09 100644
--- a/command/agent/health_endpoint_test.go
+++ b/command/agent/health_endpoint_test.go
@@ -28,6 +28,8 @@ func TestHealthChecksInState(t *testing.T) {
 	if err != nil {
 		t.Fatalf("err: %v", err)
 	}
+
+	// TODO: Failing
 	assertIndex(t, resp)
 
 	// Should be 1 health check for the server
@@ -105,6 +107,7 @@ func TestHealthServiceChecks(t *testing.T) {
 	// Should be 1 health check for consul
 	nodes := obj.(structs.HealthChecks)
 	if len(nodes) != 1 {
+		// TODO: Failing
 		t.Fatalf("bad: %v", obj)
 	}
 }
@@ -127,6 +130,8 @@ func TestHealthServiceNodes(t *testing.T) {
 	if err != nil {
 		t.Fatalf("err: %v", err)
 	}
+
+	// TODO: Failing
 	assertIndex(t, resp)
 
 	// Should be 1 health check for consul
@@ -172,6 +177,8 @@ func TestHealthServiceNodes_PassingFilter(t *testing.T) {
 	if err != nil {
 		t.Fatalf("err: %v", err)
 	}
+
+	// TODO: Failing
 	assertIndex(t, resp)
 
 	// Should be 0 health check for consul
diff --git a/command/agent/rpc_client_test.go b/command/agent/rpc_client_test.go
index 95b7292d8720..77f34632ae4c 100644
--- a/command/agent/rpc_client_test.go
+++ b/command/agent/rpc_client_test.go
@@ -243,6 +243,7 @@ OUTER1:
 		}
 	}
 	if !found {
+		// TODO: Failing
 		t.Fatalf("should log client accept")
 	}
 
diff --git a/command/agent/ui_endpoint_test.go b/command/agent/ui_endpoint_test.go
index ab734f3cb471..510ff973a326 100644
--- a/command/agent/ui_endpoint_test.go
+++ b/command/agent/ui_endpoint_test.go
@@ -100,6 +100,8 @@ func TestUiNodeInfo(t *testing.T) {
 	if err != nil {
 		t.Fatalf("err: %v", err)
 	}
+
+	// TODO: Failing
 	assertIndex(t, resp)
 
 	// Should be 1 node for the server
diff --git a/consul/catalog_endpoint_test.go b/consul/catalog_endpoint_test.go
index 3945dd51d92f..f4028b656177 100644
--- a/consul/catalog_endpoint_test.go
+++ b/consul/catalog_endpoint_test.go
@@ -540,6 +540,7 @@ func TestCatalogListServices_Timeout(t *testing.T) {
 
 	// Should block at least 100ms
 	if time.Now().Sub(start) < 100 * time.Millisecond {
+		// TODO: Failing
 		t.Fatalf("too fast")
 	}