diff --git a/.travis.yml b/.travis.yml index 575dccb0074..5748ca2ce80 100644 --- a/.travis.yml +++ b/.travis.yml @@ -68,7 +68,7 @@ jobs: go: $GO_VERSION stage: test - os: linux - env: TARGETS="-C libbeat stress-tests" + env: STRESS_TEST_OPTIONS="-timeout=20m -race -v -parallel 1" TARGETS="-C libbeat stress-tests" go: $GO_VERSION stage: test diff --git a/libbeat/publisher/pipeline/stress/gen.go b/libbeat/publisher/pipeline/stress/gen.go index e0afdfd2f0a..e3c7bd30775 100644 --- a/libbeat/publisher/pipeline/stress/gen.go +++ b/libbeat/publisher/pipeline/stress/gen.go @@ -1,7 +1,9 @@ package stress import ( + "bytes" "fmt" + "runtime/pprof" "sync" "time" @@ -25,7 +27,7 @@ var defaultGenerateConfig = generateConfig{ ACK: false, MaxEvents: 0, WaitClose: 0, - Watchdog: 1 * time.Second, + Watchdog: 2 * time.Second, } var publishModes = map[string]beat.PublishMode{ @@ -92,7 +94,7 @@ func generate( // start generator watchdog withWG(&wg, func() { last := uint64(0) - ticker := time.NewTicker(config.Watchdog) // todo: make ticker interval configurable + ticker := time.NewTicker(config.Watchdog) defer ticker.Stop() for { select { @@ -105,7 +107,11 @@ func generate( current := count.Load() if last == current { - err := fmt.Errorf("no progress in generators (last=%v, current=%v)", last, current) + // collect all active go-routines stack-traces: + var buf bytes.Buffer + pprof.Lookup("goroutine").WriteTo(&buf, 2) + + err := fmt.Errorf("no progress in generator %v (last=%v, current=%v):\n%s", id, last, current, buf.Bytes()) errors(err) } last = current diff --git a/libbeat/publisher/pipeline/stress/stress_test.go b/libbeat/publisher/pipeline/stress/stress_test.go index 7cf5955deae..93f3021420f 100644 --- a/libbeat/publisher/pipeline/stress/stress_test.go +++ b/libbeat/publisher/pipeline/stress/stress_test.go @@ -43,12 +43,22 @@ func TestPipeline(t *testing.T) { } if duration == 0 { - duration = 10 * time.Second + duration = 15 * time.Second } configTest(t, "gen", genConfigs, func(t *testing.T, gen string) { configTest(t, "pipeline", pipelineConfigs, func(t *testing.T, pipeline string) { configTest(t, "out", outConfigs, func(t *testing.T, out string) { + + if testing.Verbose() { + start := time.Now() + fmt.Printf("%v Start stress test %v\n", start.Format(time.RFC3339), t.Name()) + defer func() { + end := time.Now() + fmt.Printf("%v Finished stress test %v. Duration=%v\n", end.Format(time.RFC3339), t.Name(), end.Sub(start)) + }() + } + config, err := common.LoadFiles(gen, pipeline, out) if err != nil { t.Fatal(err) diff --git a/libbeat/scripts/Makefile b/libbeat/scripts/Makefile index 4b19aa9d5c0..b2b58b3ad9c 100755 --- a/libbeat/scripts/Makefile +++ b/libbeat/scripts/Makefile @@ -52,6 +52,7 @@ NOSETESTS_OPTIONS?=--process-timeout=$(TIMEOUT) --with-timer -v --with-xunit --x TEST_ENVIRONMENT?=false ## @testing if true, "make testsuite" runs integration tests and system tests in a dockerized test environment SYSTEM_TESTS?=false ## @testing if true, "make test" and "make testsuite" run unit tests and system tests STRESS_TESTS?=false ## @testing if true, "make test" and "make testsuite" run also run the stress tests +STRESS_TEST_OPTIONS?=-timeout=20m -race -v GOX_OS?=linux darwin windows freebsd netbsd openbsd ## @Building List of all OS to be supported by "make crosscompile". GOX_OSARCH?=!darwin/arm !darwin/arm64 !darwin/386 ## @building Space separated list of GOOS/GOARCH pairs to build by "make crosscompile". GOX_FLAGS?= ## @building Additional flags to append to the gox command used by "make crosscompile". @@ -206,7 +207,7 @@ fast-system-tests: ${BEAT_NAME}.test python-env stress-tests: ## @testing Runs the stress tests with race detector enabled stress-tests: if [ -n '${GOPACKAGES_STRESSTESTS}' ]; then \ - go test -race --tags=stresstest -v ${GOPACKAGES_STRESSTESTS}; \ + go test --tags=stresstest ${STRESS_TEST_OPTIONS} ${GOPACKAGES_STRESSTESTS}; \ fi # Run benchmark tests