test: Code coverage for e2e tests (#1475)

* Initial collecting coverage for e2e tests

* Allow to run e2e tests with and without coverage

* Add build tag to dummy test, so it is skipped in general case

* Upload coverage to codecov

test-e2e

.travis.yml

@@ -53,6 +53,7 @@ jobs:
       script: make test-cover
       after_success:
         - goveralls -coverprofile=/tmp/coverage.out -service=travis-ci
+        - bash <(curl -s https://codecov.io/bash) -f /tmp/coverage.out -F unittests
 
     - stage: Build
       script:
@@ -76,13 +77,19 @@ jobs:
     - stage: E2E
       env: VPP_VERSION=1904
       script:
-        - make e2e-tests
+        - make e2e-tests-cover
+      after_success:
+        - bash <(curl -s https://codecov.io/bash) -f /tmp/e2e-cov.out -F e2e1904
     - env: VPP_VERSION=1908
       script:
-        - make e2e-tests
+        - make e2e-tests-cover
+      after_success:
+        - bash <(curl -s https://codecov.io/bash) -f /tmp/e2e-cov.out -F e2e1908
     - env: VPP_VERSION=2001
       script:
-        - make e2e-tests
+        - make e2e-tests-cover
+      after_success:
+        - bash <(curl -s https://codecov.io/bash) -f /tmp/e2e-cov.out -F e2e2001
 
 notifications:
   slack:

Makefile

@@ -143,6 +143,11 @@ e2e-tests: ## Run end-to-end tests
 	@echo "=> running end-to-end tests"
 	VPP_IMG=$(VPP_IMG) ./tests/e2e/run_e2e.sh
 
+e2e-tests-cover: ## Run end-to-end tests with coverage
+	@echo "=> running end-to-end tests with coverage"
+	VPP_IMG=$(VPP_IMG) COVER_DIR=$(COVER_DIR) ./tests/e2e/run_e2e.sh
+	@echo "=> coverage report generated into ${COVER_DIR}/e2e-cov.out"
+
 # -------------------------------
 #  Code generation
 # -------------------------------

cmd/vpp-agent/vpp_agent_test.go

@@ -0,0 +1,9 @@
+// +build teste2e
+
+package main
+
+import "testing"
+
+func TestRunMain(t *testing.T) {
+	main()
+}

tests/e2e/e2e_test.go

@@ -33,7 +33,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/fsouza/go-dockerclient"
+	docker "github.com/fsouza/go-dockerclient"
 	"github.com/gogo/protobuf/proto"
 	"github.com/mitchellh/go-ps"
 	. "github.com/onsi/gomega"
@@ -57,6 +57,7 @@ var (
 	vppPath       = flag.String("vpp-path", "/usr/bin/vpp", "VPP program path")
 	vppConfig     = flag.String("vpp-config", "", "VPP config file")
 	vppSockAddr   = flag.String("vpp-sock-addr", "", "VPP binapi socket address")
+	covPath       = flag.String("cov", "", "Path to collect coverage data")
 	agentHTTPPort = flag.Int("agent-http-port", 9191, "VPP-Agent HTTP port")
 	agentGrpcPort = flag.Int("agent-grpc-port", 9111, "VPP-Agent GRPC port")
 	debugHTTP     = flag.Bool("debug-http", false, "Enable HTTP client debugging")
@@ -159,7 +160,14 @@ func setupE2E(t *testing.T) *testCtx {
 
 	// start the agent
 	assertProcessNotRunning(t, "vpp_agent")
-	agentCmd := startProcess(t, "VPP-Agent", nil, os.Stdout, os.Stderr, "/vpp-agent")
+
+	var agentArgs []string
+
+	if *covPath != "" {
+		e2eCovPath := fmt.Sprintf("%s/%d.out", *covPath, time.Now().Unix())
+		agentArgs = []string{"-test.coverprofile", e2eCovPath}
+	}
+	agentCmd := startProcess(t, "VPP-Agent", nil, os.Stdout, os.Stderr, "/vpp-agent", agentArgs...)
 
 	// prepare HTTP client for access to REST API of the agent
 	httpAddr := fmt.Sprintf(":%d", *agentHTTPPort)

tests/e2e/run_e2e.sh

@@ -4,10 +4,23 @@ set -euo pipefail
 echo "preparing E2E test"
 set -x
 
+args=($*)
+
 # compile vpp-agent
-go build -v -o ./tests/e2e/vpp-agent.test \
-  -ldflags "-X github.com/ligato/vpp-agent/vendor/github.com/ligato/cn-infra/agent.BuildVersion=TEST_E2E" \
-  ./cmd/vpp-agent
+if [ -z "${COVER_DIR-}" ]; then
+	go build -v -o ./tests/e2e/vpp-agent.test \
+      -ldflags "-X github.com/ligato/vpp-agent/vendor/github.com/ligato/cn-infra/agent.BuildVersion=TEST_E2E" \
+      ./cmd/vpp-agent
+else
+	if [ ! -d ${COVER_DIR}/e2e-coverage ]; then
+		mkdir ${COVER_DIR}/e2e-coverage
+	elif [ "$(ls -A ${COVER_DIR}/e2e-coverage)" ]; then
+		rm -f ${COVER_DIR}/e2e-coverage/*
+	fi
+	go test -covermode=count -coverpkg="github.com/ligato/vpp-agent/..." -c ./cmd/vpp-agent -o ./tests/e2e/vpp-agent.test -tags teste2e
+	DOCKER_ARGS="${DOCKER_ARGS-} -v ${COVER_DIR}/e2e-coverage:${COVER_DIR}/e2e-coverage"
+	args+=("-cov=${COVER_DIR}/e2e-coverage")
+fi
 
 # compile e2e test suite
 go test -c -o ./tests/e2e/e2e.test ./tests/e2e
@@ -36,6 +49,12 @@ cleanup() {
 	set -x
 	docker stop -t 2 "$cid" >/dev/null
 	docker rm "$cid" >/dev/null
+
+	# merge coverage
+	if [ ! -z "${COVER_DIR-}" ]; then
+		go get github.com/wadey/gocovmerge
+		find ${COVER_DIR}/e2e-coverage -type f | xargs gocovmerge > ${COVER_DIR}/e2e-cov.out
+	fi
 }
 
 vppver=$(docker exec -i "$cid" dpkg-query -f '${Version}' -W vpp)
@@ -47,7 +66,7 @@ echo -e " E2E TEST - VPP \e[1;33m${vppver}\e[0m"
 echo "============================================================="
 
 # run e2e test
-if docker exec -i "$cid" /e2e.test -test.v $*; then
+if docker exec -i "$cid" /e2e.test -test.v ${args[@]}; then
 	echo >&2 "-------------------------------------------------------------"
 	echo >&2 -e " \e[32mPASSED\e[0m (took: ${SECONDS}s)"
 	echo >&2 "-------------------------------------------------------------"