From 71d7c85b6be3db4f9007692835742dfc56ce219c Mon Sep 17 00:00:00 2001 From: Anthony Romano Date: Thu, 4 May 2017 18:24:59 -0700 Subject: [PATCH 1/3] expect: reload DEBUG_EXPECT for each process Lets e2e test cases selectively turn on expect debugging to get full application output written to stdout. --- pkg/expect/expect.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/expect/expect.go b/pkg/expect/expect.go index a58121ccf9b..fe27ef7fe9f 100644 --- a/pkg/expect/expect.go +++ b/pkg/expect/expect.go @@ -44,8 +44,6 @@ type ExpectProcess struct { StopSignal os.Signal } -var printDebugLines = os.Getenv("EXPECT_DEBUG") != "" - // NewExpect creates a new process for expect testing. func NewExpect(name string, arg ...string) (ep *ExpectProcess, err error) { // if env[] is nil, use current system env @@ -75,6 +73,7 @@ func NewExpectWithEnv(name string, args []string, env []string) (ep *ExpectProce func (ep *ExpectProcess) read() { defer ep.wg.Done() + printDebugLines := os.Getenv("EXPECT_DEBUG") != "" r := bufio.NewReader(ep.fpty) for ep.err == nil { ep.ptyMu.Lock() From b9f5a00b13536dd2cc6a06fe21abb99edece997c Mon Sep 17 00:00:00 2001 From: Anthony Romano Date: Thu, 4 May 2017 18:36:17 -0700 Subject: [PATCH 2/3] e2e: more debugging output for lock and elect etcdctl tests Meant to debug #6464 and #6934 Dumps the output from the etcd/etcdctl servers and SIGQUITs to get a golang backtrace in case of a hanged process. --- e2e/ctl_v3_elect_test.go | 13 ++++++++++++- e2e/ctl_v3_lock_test.go | 36 +++++++++++++++++++++++++++++++++++- e2e/etcd_test.go | 8 ++++++++ 3 files changed, 55 insertions(+), 2 deletions(-) diff --git a/e2e/ctl_v3_elect_test.go b/e2e/ctl_v3_elect_test.go index 3da54278099..58539015a97 100644 --- a/e2e/ctl_v3_elect_test.go +++ b/e2e/ctl_v3_elect_test.go @@ -23,9 +23,19 @@ import ( "github.com/coreos/etcd/pkg/expect" ) -func TestCtlV3Elect(t *testing.T) { testCtl(t, testElect) } +func TestCtlV3Elect(t *testing.T) { + oldenv := os.Getenv("EXPECT_DEBUG") + defer os.Setenv("EXPECT_DEBUG", oldenv) + os.Setenv("EXPECT_DEBUG", "1") + + testCtl(t, testElect) +} func testElect(cx ctlCtx) { + // debugging for #6934 + sig := cx.epc.withStopSignal(debugLockSignal) + defer cx.epc.withStopSignal(sig) + name := "a" holder, ch, err := ctlV3Elect(cx, name, "p1") @@ -102,6 +112,7 @@ func ctlV3Elect(cx ctlCtx, name, proposal string) (*expect.ExpectProcess, <-chan close(outc) return proc, outc, err } + proc.StopSignal = debugLockSignal go func() { s, xerr := proc.ExpectFunc(func(string) bool { return true }) if xerr != nil { diff --git a/e2e/ctl_v3_lock_test.go b/e2e/ctl_v3_lock_test.go index e380f7cd843..450805fa123 100644 --- a/e2e/ctl_v3_lock_test.go +++ b/e2e/ctl_v3_lock_test.go @@ -16,16 +16,49 @@ package e2e import ( "os" + "runtime" "strings" + "syscall" "testing" "time" "github.com/coreos/etcd/pkg/expect" ) -func TestCtlV3Lock(t *testing.T) { testCtl(t, testLock) } +// debugLockSignal forces SIGQUIT to debug etcdctl elect and lock failures +var debugLockSignal os.Signal + +func init() { + // hacks to ignore SIGQUIT debugging for some builds + switch { + case os.Getenv("COVERDIR") != "": + // SIGQUIT interferes with coverage collection + debugLockSignal = syscall.SIGTERM + case runtime.GOARCH == "ppc64le": + // ppc64le's signal handling won't kill processes with SIGQUIT + // in the same way as amd64/i386, so processes won't terminate + // as expected. Since this debugging code for CI, just ignore + // ppc64le. + debugLockSignal = syscall.SIGKILL + default: + // stack dumping OK + debugLockSignal = syscall.SIGQUIT + } +} + +func TestCtlV3Lock(t *testing.T) { + oldenv := os.Getenv("EXPECT_DEBUG") + defer os.Setenv("EXPECT_DEBUG", oldenv) + os.Setenv("EXPECT_DEBUG", "1") + + testCtl(t, testLock) +} func testLock(cx ctlCtx) { + // debugging for #6464 + sig := cx.epc.withStopSignal(debugLockSignal) + defer cx.epc.withStopSignal(sig) + name := "a" holder, ch, err := ctlV3Lock(cx, name) @@ -102,6 +135,7 @@ func ctlV3Lock(cx ctlCtx, name string) (*expect.ExpectProcess, <-chan string, er close(outc) return proc, outc, err } + proc.StopSignal = debugLockSignal go func() { s, xerr := proc.ExpectFunc(func(string) bool { return true }) if xerr != nil { diff --git a/e2e/etcd_test.go b/e2e/etcd_test.go index c4efb3f8c28..eedb80a4973 100644 --- a/e2e/etcd_test.go +++ b/e2e/etcd_test.go @@ -553,3 +553,11 @@ func (epc *etcdProcessCluster) grpcEndpoints() []string { } return eps } + +func (epc *etcdProcessCluster) withStopSignal(sig os.Signal) os.Signal { + ret := epc.procs[0].proc.StopSignal + for _, p := range epc.procs { + p.proc.StopSignal = sig + } + return ret +} From 5b4677b7d7110ed411c89efa9c0d00e76f86a01b Mon Sep 17 00:00:00 2001 From: Anthony Romano Date: Thu, 11 May 2017 16:27:57 -0700 Subject: [PATCH 3/3] integration: reset default logging level in TestRestartRemoved --- integration/cluster_test.go | 2 ++ integration/logger_test.go | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/integration/cluster_test.go b/integration/cluster_test.go index e356564c7fb..5a1040adb7a 100644 --- a/integration/cluster_test.go +++ b/integration/cluster_test.go @@ -447,7 +447,9 @@ func TestRejectUnhealthyRemove(t *testing.T) { // (see https://github.com/coreos/etcd/issues/7512 for more). func TestRestartRemoved(t *testing.T) { defer testutil.AfterTest(t) + capnslog.SetGlobalLogLevel(capnslog.INFO) + defer capnslog.SetGlobalLogLevel(defaultLogLevel) // 1. start single-member cluster c := NewCluster(t, 1) diff --git a/integration/logger_test.go b/integration/logger_test.go index a71330a41a8..aecd7200044 100644 --- a/integration/logger_test.go +++ b/integration/logger_test.go @@ -16,6 +16,8 @@ package integration import "github.com/coreos/pkg/capnslog" +const defaultLogLevel = capnslog.CRITICAL + func init() { - capnslog.SetGlobalLogLevel(capnslog.CRITICAL) + capnslog.SetGlobalLogLevel(defaultLogLevel) }