Enable E2E test running on Travis CI

- We want Travis build to fail if E2E tests fail.
- Renamed "perf-test" makefile target to "e2e-test" (since these
  are not just perf tests but also other types of end-to-end tests).
- Fix TestGeneratorAndBackend to make sure it waits properly for
  the generator to produce results.

Issue: open-telemetry#68

Makefile

@@ -35,16 +35,18 @@ all-srcs:
 .PHONY: fmt-vet-lint-test
 fmt-vet-lint-test: fmt vet lint test
 
-.PHONY: perf-test
-perf-test: otelsvc
+.PHONY: e2e-test
+e2e-test: otelsvc
 	$(MAKE) -C testbed runtests
 
 .PHONY: test
 test:
 	$(GOTEST) $(GOTEST_OPT) $(ALL_PKGS)
 
 .PHONY: travis-ci
-travis-ci: fmt vet lint test-with-cover
+travis-ci: fmt vet lint test-with-cover otelsvc
+	$(MAKE) -C testbed install-tools
+	$(MAKE) -C testbed runtests
 
 .PHONY: test-with-cover
 test-with-cover:

go.mod

@@ -1,53 +1,41 @@
 module github.com/open-telemetry/opentelemetry-service
 
 require (
-	cloud.google.com/go v0.38.0 // indirect
 	contrib.go.opencensus.io/exporter/jaeger v0.1.1-0.20190430175949-e8b55949d948
 	contrib.go.opencensus.io/exporter/ocagent v0.5.0
 	contrib.go.opencensus.io/exporter/prometheus v0.1.0
 	contrib.go.opencensus.io/exporter/zipkin v0.1.1
 	contrib.go.opencensus.io/resource v0.1.1
-	github.com/VividCortex/gohistogram v1.0.0 // indirect
 	github.com/apache/thrift v0.0.0-20161221203622-b2a4d4ae21c7
-	github.com/aws/aws-sdk-go v1.19.18 // indirect
-	github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b // indirect
 	github.com/census-instrumentation/opencensus-proto v0.2.0
 	github.com/go-kit/kit v0.8.0
-	github.com/gogo/googleapis v1.2.0 // indirect
 	github.com/golang/protobuf v1.3.1
 	github.com/google/go-cmp v0.3.0
 	github.com/gorilla/mux v1.6.2
-	github.com/grpc-ecosystem/grpc-gateway v1.8.5
-	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway v1.9.0
 	github.com/jaegertracing/jaeger v1.9.0
 	github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024
 	github.com/omnition/scribe-go v0.0.0-20190131012523-9e3c68f31124
-	github.com/opentracing/opentracing-go v1.1.0 // indirect
+	github.com/open-telemetry/opentelemetry-service/testbed v0.0.0-20190628004302-f28c6b82c36f // indirect
 	github.com/openzipkin/zipkin-go v0.1.6
 	github.com/orijtech/prometheus-go-metrics-exporter v0.0.3-0.20190313163149-b321c5297f60
 	github.com/pkg/errors v0.8.0
-	github.com/prashantv/protectmem v0.0.0-20171002184600-e20412882b3a // indirect
-	github.com/prometheus/client_golang v0.9.2
-	github.com/prometheus/common v0.0.0-20181126121408-4724e9255275
-	github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1
+	github.com/prometheus/client_golang v0.9.3
+	github.com/prometheus/common v0.4.0
+	github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084
 	github.com/prometheus/prometheus v0.0.0-20190131111325-62e591f928dd
 	github.com/rs/cors v1.6.0
 	github.com/soheilhy/cmux v0.1.4
-	github.com/spf13/cast v1.2.0
+	github.com/spf13/cast v1.3.0
 	github.com/spf13/cobra v0.0.3
-	github.com/spf13/viper v1.2.1
-	github.com/streadway/quantile v0.0.0-20150917103942-b0c588724d25 // indirect
+	github.com/spf13/viper v1.4.0
 	github.com/stretchr/testify v1.3.0
-	github.com/uber-go/atomic v1.3.2 // indirect
-	github.com/uber/jaeger-client-go v2.16.0+incompatible // indirect
+	github.com/uber-go/atomic v1.4.0 // indirect
 	github.com/uber/jaeger-lib v2.0.0+incompatible
 	github.com/uber/tchannel-go v1.10.0
 	go.opencensus.io v0.22.0
-	go.uber.org/atomic v1.3.2 // indirect
-	go.uber.org/multierr v1.1.0 // indirect
-	go.uber.org/zap v1.9.1
+	go.uber.org/zap v1.10.0
 	golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3
-	golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a // indirect
 	google.golang.org/api v0.5.0
 	google.golang.org/grpc v1.21.0
 	gopkg.in/yaml.v2 v2.2.2

testbed/Makefile

@@ -17,8 +17,7 @@ test:
 
 .PHONY: runtests
 runtests: test
-	cd tests && TESTBED_CONFIG=local.yaml go test -v 2>&1 | tee results/testoutput.log
-	cd tests && mkdir -p results/junit && go-junit-report < results/testoutput.log > results/junit/results.xml
+	./runtests.sh
 
 .PHONY: fmt
 fmt:

testbed/go.sum

@@ -190,8 +190,6 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW
 github.com/oklog/oklog v0.0.0-20170918173356-f857583a70c3/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/olekukonko/tablewriter v0.0.0-20180912035003-be2c049b30cc/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
-github.com/omnition/opencensus-proto v0.2.2-gogo3-unary h1:N/Eb32LTEBVpUV2tvxTNyi6bznvpfn5SBaxBAGQgI2Q=
-github.com/omnition/opencensus-proto v0.2.2-gogo3-unary/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/omnition/scribe-go v0.0.0-20190131012523-9e3c68f31124/go.mod h1:GnPmaNTr3pdt/V0JmVNVgDq+JEMb/oXxNlsG+pN6gg4=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
@@ -320,6 +318,7 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3 h1:XQyxROzUlZH+WIQwySDgnISgOivlhjIEwaQaJEJrrN0=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -368,6 +367,7 @@ golang.org/x/tools v0.0.0-20181023010539-40a48ad93fbe/go.mod h1:n7NCudcB/nEzxVGm
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138 h1:H3uGjxCR/6Ds0Mjgyp7LMK81+LvmbvWWEnJhzk1Pi9E=
 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 google.golang.org/api v0.0.0-20180506000402-20530fd5d65a/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
 google.golang.org/api v0.4.0 h1:KKgc1aqhV8wDPbDzlDtpvyjZFY3vjz85FP7p4wcQUyI=

testbed/runtests.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -e
+
+cd tests
+
+SED="sed --unbuffered"
+
+COLOR_PASS=$(printf "\033[32mPASS\033[0m")
+COLOR_FAIL=$(printf "\033[31mFAIL\033[0m")
+COLORIZE="${SED} 's/PASS/${COLOR_PASS}/' | ${SED} 's/FAIL/${COLOR_FAIL}/'"
+
+bash -c "TESTBED_CONFIG=local.yaml go test -v 2>&1 | tee results/testoutput.log | ${COLORIZE}"
+
+testStatus=${PIPESTATUS[0]}
+
+mkdir -p results/junit
+go-junit-report < results/testoutput.log > results/junit/results.xml
+
+bash -c "cat results/TESTRESULTS.md | ${COLORIZE}"
+
+exit ${testStatus}

testbed/testbed/.gitignore

@@ -0,0 +1 @@
+mockbackend.log

testbed/testbed/load_generator.go

@@ -1,3 +1,17 @@
+// Copyright 2019, OpenCensus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package testbed
 
 import (
@@ -98,6 +112,10 @@ func (lg *LoadGenerator) GetStats() string {
 	return fmt.Sprintf("Sent:%5d spans", atomic.LoadUint64(&lg.SpansSent))
 }
 
+func (lg *LoadGenerator) GetSpansSent() uint64 {
+	return atomic.LoadUint64(&lg.SpansSent)
+}
+
 func (lg *LoadGenerator) generate() {
 	// Indicate that generation is done at the end
 	defer lg.stopWait.Done()

testbed/testbed/mock_backend.go

@@ -1,3 +1,17 @@
+// Copyright 2019, OpenCensus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package testbed
 
 import (

testbed/testbed/mock_backend_test.go

@@ -1,3 +1,17 @@
+// Copyright 2019, OpenCensus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package testbed
 
 import (
@@ -24,17 +38,42 @@ func TestGeneratorAndBackend(t *testing.T) {
 
 	assert.EqualValues(t, 0, lg.SpansSent)
 
-	// Generate for about 10ms at 1000 SPS
+	// Generate at 1000 SPS
 	lg.Start(LoadOptions{SpansPerSecond: 1000})
 
-	time.Sleep(time.Millisecond * 10)
+	// Wait until at least 50 spans are sent
+	WaitFor(t, func() bool { return lg.GetSpansSent() > 50 }, "SpansSent > 50")
 
 	lg.Stop()
 
-	// Presumably should have generated something. If not then the testbed is very slow
-	// so we will consider it a failure.
-	assert.True(t, lg.SpansSent > 0)
-
 	// The backend should receive everything generated.
 	assert.Equal(t, lg.SpansSent, mb.SpansReceived())
 }
+
+// WaitFor the specific condition for up to 5 seconds. Records a test error
+// if condition does not become true.
+func WaitFor(t *testing.T, cond func() bool, errMsg ...interface{}) bool {
+	startTime := time.Now()
+
+	// Start with 5 ms waiting interval between condition re-evaluation.
+	waitInterval := time.Millisecond * 5
+
+	for {
+		time.Sleep(waitInterval)
+
+		// Increase waiting interval exponentially up to 500 ms.
+		if waitInterval < time.Millisecond*500 {
+			waitInterval = waitInterval * 2
+		}
+
+		if cond() {
+			return true
+		}
+
+		if time.Since(startTime) > time.Second*5 {
+			// Waited too long
+			t.Error("Time out waiting for", errMsg)
+			return false
+		}
+	}
+}

testbed/testbed/results.go

@@ -1,3 +1,17 @@
+// Copyright 2019, OpenCensus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package testbed
 
 import (
@@ -45,8 +59,8 @@ func (r *Results) Init(resultsDir string) {
 	_, _ = io.WriteString(r.resultsFile,
 		"# Test Results\n"+
 			fmt.Sprintf("Started: %s\n\n", time.Now().Format(time.RFC1123Z))+
-			"Test|Result|Duration|CPU Avg%|CPU Max%|RAM Avg MiB|RAM Max MiB|Sent Spans|Received Spans\n"+
-			"----|------|-------:|-------:|-------:|----------:|----------:|---------:|-------------:\n")
+			"Test                                    |Result|Duration|CPU Avg%|CPU Max%|RAM Avg MiB|RAM Max MiB|Sent Spans|Received Spans\n"+
+			"----------------------------------------|------|-------:|-------:|-------:|----------:|----------:|---------:|-------------:\n")
 }
 
 // Save the total results and close the file.
@@ -59,7 +73,7 @@ func (r *Results) Save() {
 // Add results for one test.
 func (r *Results) Add(testName string, result *TestResult) {
 	_, _ = io.WriteString(r.resultsFile,
-		fmt.Sprintf("%s|%s|%.0fs|%.1f|%.1f|%d|%d|%d|%d\n",
+		fmt.Sprintf("%-40s|%-6s|%7.0fs|%8.1f|%8.1f|%11d|%11d|%10d|%14d\n",
 			result.testName,
 			result.result,
 			result.duration.Seconds(),

testbed/testbed/test_case.go

@@ -197,9 +197,9 @@ func (tc *TestCase) Stop() {
 
 	var result string
 	if tc.t.Failed() {
-		result = "Failed"
+		result = "FAIL"
 	} else {
-		result = "Passed"
+		result = "PASS"
 	}
 
 	results.Add(tc.t.Name(), &TestResult{

testbed/tests/.gitignore

@@ -0,0 +1 @@
+results/*

testbed/tests/perf_test.go

@@ -155,7 +155,7 @@ func Test1000SPSWithAttributes(t *testing.T) {
 			defer tc.Stop()
 
 			tc.SetExpectedMaxCPU(test.expectedCPU)
-			tc.SetExpectedMaxRAM(70)
+			tc.SetExpectedMaxRAM(10)
 
 			tc.StartBackend(testbed.BackendOC)
 			tc.StartAgent()