Added memory ballast e2e behavior test (#85)

Added a new test that verifies that the ballast behaves only counts to
virtual memory and is not actually allocated.

Also added `tests/results/BASELINE.md` to act as the baseline test
results file. The TESTRESULTS.md file was modified on every run and it
would change in every PR from every contributor. It added additional
work when one didn't want to include it in a PR which was most of the
time. Only time it should be updated is when someone adds or updates a
new perf test.

testbed/testbed/options.go

@@ -0,0 +1,35 @@
+// Copyright 2019, OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package tests contains test cases. To run the tests go to tests directory and run:
+// TESTBED_CONFIG=local.yaml go test -v
+
+package testbed
+
+// TestCaseOption defines a TestCase option
+type TestCaseOption struct {
+	option func(t *TestCase)
+}
+
+// Apply takes a TestCase and runs the option function on it
+func (o TestCaseOption) Apply(t *TestCase) {
+	o.option(t)
+}
+
+// WithSkipResults option disables writing out results file for a TestCase
+func WithSkipResults() TestCaseOption {
+	return TestCaseOption{func(t *TestCase) {
+		t.skipResults = true
+	}}
+}

testbed/testbed/test_case.go

@@ -32,6 +32,9 @@ type TestCase struct {
 	// Directory where test case results and logs will be written.
 	resultDir string
 
+	// does not write out results when set to true
+	skipResults bool
+
 	// Agent config file path.
 	agentConfigFile string
 
@@ -60,14 +63,18 @@ type TestCase struct {
 const mibibyte = 1024 * 1024
 
 // NewTestCase creates a new TestCase. It expected agent-config.yaml in the specified directory.
-func NewTestCase(t *testing.T) *TestCase {
+func NewTestCase(t *testing.T, opts ...TestCaseOption) *TestCase {
 	tc := TestCase{}
 
 	tc.t = t
 	tc.ErrorSignal = make(chan struct{})
 	tc.doneSignal = make(chan struct{})
 	tc.startTime = time.Now()
 
+	for _, opt := range opts {
+		opt.Apply(&tc)
+	}
+
 	var err error
 	tc.resultDir, err = filepath.Abs(path.Join("results", t.Name()))
 	if err != nil {
@@ -123,14 +130,16 @@ func (tc *TestCase) SetExpectedMaxRAM(ramMiB uint32) {
 
 // StartAgent starts the agent and redirects its standard output and standard error
 // to "agent.log" file located in the test directory.
-func (tc *TestCase) StartAgent() {
+func (tc *TestCase) StartAgent(args ...string) {
+	args = append(args, "--config")
+	args = append(args, tc.agentConfigFile)
 	logFileName := tc.composeTestResultFileName("agent.log")
 
 	err := tc.agentProc.start(startParams{
 		name:         "Agent",
 		logFilePath:  logFileName,
 		cmd:          testBedConfig.Agent,
-		cmdArgs:      []string{"--config", tc.agentConfigFile},
+		cmdArgs:      args, 
 		resourceSpec: &tc.resourceSpec,
 	})
 
@@ -180,6 +189,16 @@ func (tc *TestCase) StopBackend() {
 	tc.MockBackend.Stop()
 }
 
+// AgentMemoryInfo returns raw memory info struct about the agent 
+// as returned by github.com/shirou/gopsutil/process
+func (tc *TestCase) AgentMemoryInfo() (uint32, uint32, error) {
+	stat, err := tc.agentProc.processMon.MemoryInfo()
+	if err != nil {
+		return 0, 0, err
+	}
+	return uint32(stat.RSS / mibibyte), uint32(stat.VMS / mibibyte), nil
+}
+
 // Stop stops the load generator, the agent and the backend.
 func (tc *TestCase) Stop() {
 	// Stop all components
@@ -190,6 +209,10 @@ func (tc *TestCase) Stop() {
 	// Stop logging
 	close(tc.doneSignal)
 
+	if tc.skipResults {
+		return
+	}
+
 	// Report test results
 
 	rc := tc.agentProc.GetTotalConsumption()
@@ -238,11 +261,11 @@ func (tc *TestCase) Sleep(d time.Duration) {
 	}
 }
 
-// WaitFor the specific condition for up to 10 seconds. Records a test error
+// WaitForN the specific condition for up to a specified duration. Records a test error
 // if time is out and condition does not become true. If error is signalled
 // while waiting the function will return false, but will not record additional
 // test error (we assume that signalled error is already recorded in indicateError()).
-func (tc *TestCase) WaitFor(cond func() bool, errMsg ...interface{}) bool {
+func (tc *TestCase) WaitForN(cond func() bool, duration time.Duration, errMsg ...interface{}) bool {
 	startTime := time.Now()
 
 	// Start with 5 ms waiting interval between condition re-evaluation.
@@ -264,14 +287,19 @@ func (tc *TestCase) WaitFor(cond func() bool, errMsg ...interface{}) bool {
 			waitInterval = waitInterval * 2
 		}
 
-		if time.Since(startTime) > time.Second*10 {
+		if time.Since(startTime) > duration {
 			// Waited too long
 			tc.t.Error("Time out waiting for", errMsg)
 			return false
 		}
 	}
 }
 
+// WaitFor is like WaitForN but with a fixed duration of 10 seconds
+func (tc *TestCase) WaitFor(cond func() bool, errMsg ...interface{}) bool {
+	return tc.WaitForN(cond, time.Second * 10, errMsg...)
+}
+
 func (tc *TestCase) indicateError(err error) {
 	// Print to log for visibility
 	log.Print(err.Error())

testbed/tests/.gitignore

@@ -1 +1,2 @@
 results/*
+!results/BASELINE.md

testbed/tests/e2e_test.go

@@ -0,0 +1,60 @@
+// Copyright 2019, OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package tests contains test cases. To run the tests go to tests directory and run:
+// TESTBED_CONFIG=local.yaml go test -v
+
+package tests
+
+import (
+	"fmt"
+	"testing"
+	"time"
+	"strconv"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/open-telemetry/opentelemetry-service/testbed/testbed"
+)
+
+func TestBallastMemory(t *testing.T) {
+	tests := []struct{
+		ballastSize uint32
+		maxRSS uint32
+	}{
+		{100, 50},
+		{500, 70},
+		{1000, 100},
+	}
+
+	for _, test := range tests {
+		tc := testbed.NewTestCase(t, testbed.WithSkipResults())
+		tc.SetExpectedMaxRAM(test.maxRSS)
+
+		tc.StartAgent("--mem-ballast-size-mib", strconv.Itoa(int(test.ballastSize)))
+
+		var rss, vms uint32
+		// It is possible that the process is not ready or the ballast code path 
+		// is not hit immediately so we give the process up to a couple of seconds
+		// to fire up and setup ballast. 2 seconds is a long time for this case but
+		// it is short enough to not be annoying if the test fails repeatedly
+		tc.WaitForN(func() bool {
+			rss, vms, _ = tc.AgentMemoryInfo()
+			return vms > test.ballastSize
+		}, time.Second * 2, "VMS must be greater than %d", test.ballastSize)
+
+		assert.True(t, rss <= test.maxRSS, fmt.Sprintf("RSS must be less than or equal to %d", test.maxRSS))
+		tc.Stop()
+	}
+}