Improve pipeline stress test

- Improve stability by increasing test duration, timeouts and watchdog
  timer
- Add test start/stop messages if run with `-v`. Help with travis timing
  out tests with 10min without any output
- Add all active go-routine stack traces to errors

.travis.yml

@@ -68,7 +68,7 @@ jobs:
  go: $GO_VERSION
  stage: test
  - os: linux
- env: TARGETS="-C libbeat stress-tests"
+ env: STRESS_TEST_OPTIONS="-timeout=20m -race -v -parallel 1" TARGETS="-C libbeat stress-tests"
  go: $GO_VERSION
  stage: test
 

libbeat/publisher/pipeline/stress/gen.go

@@ -1,7 +1,9 @@
 package stress
 
 import (
+ "bytes"
  "fmt"
+ "runtime/pprof"
  "sync"
  "time"
 
@@ -25,7 +27,7 @@ var defaultGenerateConfig = generateConfig{
  ACK: false,
  MaxEvents: 0,
  WaitClose: 0,
- Watchdog: 1 * time.Second,
+ Watchdog: 2 * time.Second,
 }
 
 var publishModes = map[string]beat.PublishMode{
@@ -92,7 +94,7 @@ func generate(
  // start generator watchdog
  withWG(&wg, func() {
  last := uint64(0)
- ticker := time.NewTicker(config.Watchdog) // todo: make ticker interval configurable
+ ticker := time.NewTicker(config.Watchdog)
  defer ticker.Stop()
  for {
  select {
@@ -105,7 +107,11 @@ func generate(
 
  current := count.Load()
  if last == current {
- err := fmt.Errorf("no progress in generators (last=%v, current=%v)", last, current)
+ // collect all active go-routines stack-traces:
+ var buf bytes.Buffer
+ pprof.Lookup("goroutine").WriteTo(&buf, 2)
+
+ err := fmt.Errorf("no progress in generator %v (last=%v, current=%v):\n%s", id, last, current, buf.Bytes())
  errors(err)
  }
  last = current

libbeat/publisher/pipeline/stress/stress_test.go

@@ -43,12 +43,22 @@ func TestPipeline(t *testing.T) {
  }
 
  if duration == 0 {
- duration = 10 * time.Second
+ duration = 15 * time.Second
  }
 
  configTest(t, "gen", genConfigs, func(t *testing.T, gen string) {
  configTest(t, "pipeline", pipelineConfigs, func(t *testing.T, pipeline string) {
  configTest(t, "out", outConfigs, func(t *testing.T, out string) {
+
+ if testing.Verbose() {
+ start := time.Now()
+ fmt.Printf("%v Start stress test %v\n", start.Format(time.RFC3339), t.Name())
+ defer func() {
+ end := time.Now()
+ fmt.Printf("%v Finished stress test %v. Duration=%v\n", end.Format(time.RFC3339), t.Name(), end.Sub(start))
+ }()
+ }
+
  config, err := common.LoadFiles(gen, pipeline, out)
  if err != nil {
  t.Fatal(err)

libbeat/scripts/Makefile

@@ -52,6 +52,7 @@ NOSETESTS_OPTIONS?=--process-timeout=$(TIMEOUT) --with-timer -v --with-xunit --x
 TEST_ENVIRONMENT?=false ## @testing if true, "make testsuite" runs integration tests and system tests in a dockerized test environment
 SYSTEM_TESTS?=false ## @testing if true, "make test" and "make testsuite" run unit tests and system tests
 STRESS_TESTS?=false ## @testing if true, "make test" and "make testsuite" run also run the stress tests
+STRESS_TEST_OPTIONS?=-timeout=20m -race -v
 GOX_OS?=linux darwin windows freebsd netbsd openbsd ## @Building List of all OS to be supported by "make crosscompile".
 GOX_OSARCH?=!darwin/arm !darwin/arm64 !darwin/386 ## @building Space separated list of GOOS/GOARCH pairs to build by "make crosscompile".
 GOX_FLAGS?= ## @building Additional flags to append to the gox command used by "make crosscompile".
@@ -206,7 +207,7 @@ fast-system-tests: ${BEAT_NAME}.test python-env
 stress-tests: ## @testing Runs the stress tests with race detector enabled
 stress-tests:
  if [ -n '${GOPACKAGES_STRESSTESTS}' ]; then \
- go test -race --tags=stresstest -v ${GOPACKAGES_STRESSTESTS}; \
+ go test --tags=stresstest ${STRESS_TEST_OPTIONS} ${GOPACKAGES_STRESSTESTS}; \
  fi
 
 # Run benchmark tests