From e5d83762ef5b9e95054d6b1cfd274e704f853422 Mon Sep 17 00:00:00 2001
From: Tigran Najaryan <tigran@omnition.io>
Date: Fri, 28 Jun 2019 08:21:32 -0400
Subject: [PATCH] Enable E2E test running on Travis CI

- We want Travis build to fail if E2E tests fail.
- Renamed "perf-test" makefile target to "e2e-test" (since these
  are not just perf tests but also other types of end-to-end tests).
- Fix TestGeneratorAndBackend to make sure it waits properly for
  the generator to produce results.

Issue: https://github.com/open-telemetry/opentelemetry-service/issues/68
---
 Makefile                             |  7 +++---
 testbed/testbed/.gitignore           |  1 +
 testbed/testbed/load_generator.go    |  4 +++
 testbed/testbed/mock_backend_test.go | 37 +++++++++++++++++++++++-----
 testbed/tests/.gitignore             |  1 +
 5 files changed, 41 insertions(+), 9 deletions(-)
 create mode 100644 testbed/testbed/.gitignore
 create mode 100644 testbed/tests/.gitignore

diff --git a/Makefile b/Makefile
index f167a42d6bb..39a73d6caf5 100644
--- a/Makefile
+++ b/Makefile
@@ -35,8 +35,8 @@ all-srcs:
 .PHONY: fmt-vet-lint-test
 fmt-vet-lint-test: fmt vet lint test
 
-.PHONY: perf-test
-perf-test: otelsvc
+.PHONY: e2e-test
+e2e-test: otelsvc
 	$(MAKE) -C testbed runtests
 
 .PHONY: test
@@ -44,7 +44,8 @@ test:
 	$(GOTEST) $(GOTEST_OPT) $(ALL_PKGS)
 
 .PHONY: travis-ci
-travis-ci: fmt vet lint test-with-cover
+travis-ci: fmt vet lint test-with-cover otelsvc
+	$(MAKE) -C testbed runtests
 
 .PHONY: test-with-cover
 test-with-cover:
diff --git a/testbed/testbed/.gitignore b/testbed/testbed/.gitignore
new file mode 100644
index 00000000000..9835eb192b0
--- /dev/null
+++ b/testbed/testbed/.gitignore
@@ -0,0 +1 @@
+mockbackend.log
diff --git a/testbed/testbed/load_generator.go b/testbed/testbed/load_generator.go
index 5cad1bc9bd9..3c2402c5549 100644
--- a/testbed/testbed/load_generator.go
+++ b/testbed/testbed/load_generator.go
@@ -98,6 +98,10 @@ func (lg *LoadGenerator) GetStats() string {
 	return fmt.Sprintf("Sent:%5d spans", atomic.LoadUint64(&lg.SpansSent))
 }
 
+func (lg *LoadGenerator) GetSpansSent() uint64 {
+	return atomic.LoadUint64(&lg.SpansSent)
+}
+
 func (lg *LoadGenerator) generate() {
 	// Indicate that generation is done at the end
 	defer lg.stopWait.Done()
diff --git a/testbed/testbed/mock_backend_test.go b/testbed/testbed/mock_backend_test.go
index f1259b45877..bfc60facf8b 100644
--- a/testbed/testbed/mock_backend_test.go
+++ b/testbed/testbed/mock_backend_test.go
@@ -24,17 +24,42 @@ func TestGeneratorAndBackend(t *testing.T) {
 
 	assert.EqualValues(t, 0, lg.SpansSent)
 
-	// Generate for about 10ms at 1000 SPS
+	// Generate at 1000 SPS
 	lg.Start(LoadOptions{SpansPerSecond: 1000})
 
-	time.Sleep(time.Millisecond * 10)
+	// Wait until at least 50 spans are sent
+	WaitFor(t, func() bool { return lg.GetSpansSent() > 50 }, "SpansSent > 50")
 
 	lg.Stop()
 
-	// Presumably should have generated something. If not then the testbed is very slow
-	// so we will consider it a failure.
-	assert.True(t, lg.SpansSent > 0)
-
 	// The backend should receive everything generated.
 	assert.Equal(t, lg.SpansSent, mb.SpansReceived())
 }
+
+// WaitFor the specific condition for up to 5 seconds. Records a test error
+// if condition does not become true.
+func WaitFor(t *testing.T, cond func() bool, errMsg ...interface{}) bool {
+	startTime := time.Now()
+
+	// Start with 5 ms waiting interval between condition re-evaluation.
+	waitInterval := time.Millisecond * 5
+
+	for {
+		time.Sleep(waitInterval)
+
+		// Increase waiting interval exponentially up to 500 ms.
+		if waitInterval < time.Millisecond*500 {
+			waitInterval = waitInterval * 2
+		}
+
+		if cond() {
+			return true
+		}
+
+		if time.Since(startTime) > time.Second*5 {
+			// Waited too long
+			t.Error("Time out waiting for", errMsg)
+			return false
+		}
+	}
+}
diff --git a/testbed/tests/.gitignore b/testbed/tests/.gitignore
new file mode 100644
index 00000000000..484ab7e5c61
--- /dev/null
+++ b/testbed/tests/.gitignore
@@ -0,0 +1 @@
+results/*