From 6b130d62dbaa85fc98b17630ad6717771aa9544b Mon Sep 17 00:00:00 2001 From: Owais Lone Date: Wed, 3 Jul 2019 03:29:10 +0530 Subject: [PATCH] Added memory ballast e2e behavior test (#85) Added a new test that verifies that the ballast behaves only counts to virtual memory and is not actually allocated. Also added `tests/results/BASELINE.md` to act as the baseline test results file. The TESTRESULTS.md file was modified on every run and it would change in every PR from every contributor. It added additional work when one didn't want to include it in a PR which was most of the time. Only time it should be updated is when someone adds or updates a new perf test. --- testbed/testbed/options.go | 35 +++++++++++ testbed/testbed/test_case.go | 40 +++++++++++-- testbed/tests/.gitignore | 1 + testbed/tests/e2e_test.go | 60 +++++++++++++++++++ .../results/{TESTRESULTS.md => BASELINE.md} | 0 5 files changed, 130 insertions(+), 6 deletions(-) create mode 100644 testbed/testbed/options.go create mode 100644 testbed/tests/e2e_test.go rename testbed/tests/results/{TESTRESULTS.md => BASELINE.md} (100%) diff --git a/testbed/testbed/options.go b/testbed/testbed/options.go new file mode 100644 index 00000000000..f195e5665b2 --- /dev/null +++ b/testbed/testbed/options.go @@ -0,0 +1,35 @@ +// Copyright 2019, OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package tests contains test cases. To run the tests go to tests directory and run: +// TESTBED_CONFIG=local.yaml go test -v + +package testbed + +// TestCaseOption defines a TestCase option +type TestCaseOption struct { + option func(t *TestCase) +} + +// Apply takes a TestCase and runs the option function on it +func (o TestCaseOption) Apply(t *TestCase) { + o.option(t) +} + +// WithSkipResults option disables writing out results file for a TestCase +func WithSkipResults() TestCaseOption { + return TestCaseOption{func(t *TestCase) { + t.skipResults = true + }} +} diff --git a/testbed/testbed/test_case.go b/testbed/testbed/test_case.go index 472742c147d..68b411178cb 100644 --- a/testbed/testbed/test_case.go +++ b/testbed/testbed/test_case.go @@ -32,6 +32,9 @@ type TestCase struct { // Directory where test case results and logs will be written. resultDir string + // does not write out results when set to true + skipResults bool + // Agent config file path. agentConfigFile string @@ -60,7 +63,7 @@ type TestCase struct { const mibibyte = 1024 * 1024 // NewTestCase creates a new TestCase. It expected agent-config.yaml in the specified directory. -func NewTestCase(t *testing.T) *TestCase { +func NewTestCase(t *testing.T, opts ...TestCaseOption) *TestCase { tc := TestCase{} tc.t = t @@ -68,6 +71,10 @@ func NewTestCase(t *testing.T) *TestCase { tc.doneSignal = make(chan struct{}) tc.startTime = time.Now() + for _, opt := range opts { + opt.Apply(&tc) + } + var err error tc.resultDir, err = filepath.Abs(path.Join("results", t.Name())) if err != nil { @@ -123,14 +130,16 @@ func (tc *TestCase) SetExpectedMaxRAM(ramMiB uint32) { // StartAgent starts the agent and redirects its standard output and standard error // to "agent.log" file located in the test directory. -func (tc *TestCase) StartAgent() { +func (tc *TestCase) StartAgent(args ...string) { + args = append(args, "--config") + args = append(args, tc.agentConfigFile) logFileName := tc.composeTestResultFileName("agent.log") err := tc.agentProc.start(startParams{ name: "Agent", logFilePath: logFileName, cmd: testBedConfig.Agent, - cmdArgs: []string{"--config", tc.agentConfigFile}, + cmdArgs: args, resourceSpec: &tc.resourceSpec, }) @@ -180,6 +189,16 @@ func (tc *TestCase) StopBackend() { tc.MockBackend.Stop() } +// AgentMemoryInfo returns raw memory info struct about the agent +// as returned by github.com/shirou/gopsutil/process +func (tc *TestCase) AgentMemoryInfo() (uint32, uint32, error) { + stat, err := tc.agentProc.processMon.MemoryInfo() + if err != nil { + return 0, 0, err + } + return uint32(stat.RSS / mibibyte), uint32(stat.VMS / mibibyte), nil +} + // Stop stops the load generator, the agent and the backend. func (tc *TestCase) Stop() { // Stop all components @@ -190,6 +209,10 @@ func (tc *TestCase) Stop() { // Stop logging close(tc.doneSignal) + if tc.skipResults { + return + } + // Report test results rc := tc.agentProc.GetTotalConsumption() @@ -238,11 +261,11 @@ func (tc *TestCase) Sleep(d time.Duration) { } } -// WaitFor the specific condition for up to 10 seconds. Records a test error +// WaitForN the specific condition for up to a specified duration. Records a test error // if time is out and condition does not become true. If error is signalled // while waiting the function will return false, but will not record additional // test error (we assume that signalled error is already recorded in indicateError()). -func (tc *TestCase) WaitFor(cond func() bool, errMsg ...interface{}) bool { +func (tc *TestCase) WaitForN(cond func() bool, duration time.Duration, errMsg ...interface{}) bool { startTime := time.Now() // Start with 5 ms waiting interval between condition re-evaluation. @@ -264,7 +287,7 @@ func (tc *TestCase) WaitFor(cond func() bool, errMsg ...interface{}) bool { waitInterval = waitInterval * 2 } - if time.Since(startTime) > time.Second*10 { + if time.Since(startTime) > duration { // Waited too long tc.t.Error("Time out waiting for", errMsg) return false @@ -272,6 +295,11 @@ func (tc *TestCase) WaitFor(cond func() bool, errMsg ...interface{}) bool { } } +// WaitFor is like WaitForN but with a fixed duration of 10 seconds +func (tc *TestCase) WaitFor(cond func() bool, errMsg ...interface{}) bool { + return tc.WaitForN(cond, time.Second * 10, errMsg...) +} + func (tc *TestCase) indicateError(err error) { // Print to log for visibility log.Print(err.Error()) diff --git a/testbed/tests/.gitignore b/testbed/tests/.gitignore index 484ab7e5c61..a61c5ef81e8 100644 --- a/testbed/tests/.gitignore +++ b/testbed/tests/.gitignore @@ -1 +1,2 @@ results/* +!results/BASELINE.md diff --git a/testbed/tests/e2e_test.go b/testbed/tests/e2e_test.go new file mode 100644 index 00000000000..06392785e43 --- /dev/null +++ b/testbed/tests/e2e_test.go @@ -0,0 +1,60 @@ +// Copyright 2019, OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package tests contains test cases. To run the tests go to tests directory and run: +// TESTBED_CONFIG=local.yaml go test -v + +package tests + +import ( + "fmt" + "testing" + "time" + "strconv" + + "github.com/stretchr/testify/assert" + + "github.com/open-telemetry/opentelemetry-service/testbed/testbed" +) + +func TestBallastMemory(t *testing.T) { + tests := []struct{ + ballastSize uint32 + maxRSS uint32 + }{ + {100, 50}, + {500, 70}, + {1000, 100}, + } + + for _, test := range tests { + tc := testbed.NewTestCase(t, testbed.WithSkipResults()) + tc.SetExpectedMaxRAM(test.maxRSS) + + tc.StartAgent("--mem-ballast-size-mib", strconv.Itoa(int(test.ballastSize))) + + var rss, vms uint32 + // It is possible that the process is not ready or the ballast code path + // is not hit immediately so we give the process up to a couple of seconds + // to fire up and setup ballast. 2 seconds is a long time for this case but + // it is short enough to not be annoying if the test fails repeatedly + tc.WaitForN(func() bool { + rss, vms, _ = tc.AgentMemoryInfo() + return vms > test.ballastSize + }, time.Second * 2, "VMS must be greater than %d", test.ballastSize) + + assert.True(t, rss <= test.maxRSS, fmt.Sprintf("RSS must be less than or equal to %d", test.maxRSS)) + tc.Stop() + } +} \ No newline at end of file diff --git a/testbed/tests/results/TESTRESULTS.md b/testbed/tests/results/BASELINE.md similarity index 100% rename from testbed/tests/results/TESTRESULTS.md rename to testbed/tests/results/BASELINE.md