Generalize testbed

Makefile

@@ -54,11 +54,13 @@ all: checklicense impi lint misspell test otelcol
 
 .PHONY: testbed-runtests
 testbed-runtests: otelcol
+	cd ./testbed/correctness && ./runtests.sh
 	cd ./testbed/tests && ./runtests.sh
 
 .PHONY: testbed-listtests
 testbed-listtests:
-	TESTBED_CONFIG=local.yaml $(GOTEST) -v ./testbed/tests --test.list '.*'|head -n -1
+	TESTBED_CONFIG=inprocess.yaml $(GOTEST) -v ./testbed/correctness --test.list '.*'|head -n 10
+	TESTBED_CONFIG=local.yaml $(GOTEST) -v ./testbed/tests --test.list '.*'|head -n 20
 
 .PHONY: test
 test:

service/service.go

@@ -228,6 +228,15 @@ func (app *Application) RegisterZPages(mux *http.ServeMux, pathPrefix string) {
 	mux.HandleFunc(path.Join(pathPrefix, extensionzPath), app.handleExtensionzRequest)
 }
 
+func (app *Application) SignalTestComplete() {
+	defer func() {
+		if r := recover(); r != nil {
+			app.logger.Info("stopTestChan already closed")
+		}
+	}()
+	close(app.stopTestChan)
+}
+
 func (app *Application) init() error {
 	l, err := newLogger()
 	if err != nil {

service/service_test.go

@@ -34,6 +34,7 @@ import (
 	"go.opentelemetry.io/collector/component"
 	"go.opentelemetry.io/collector/config"
 	"go.opentelemetry.io/collector/config/configmodels"
+	"go.opentelemetry.io/collector/internal/version"
 	"go.opentelemetry.io/collector/service/defaultcomponents"
 	"go.opentelemetry.io/collector/testutils"
 )
@@ -74,6 +75,47 @@ func TestApplication_Start(t *testing.T) {
 	assert.Equal(t, Closed, <-app.GetStateChannel())
 }
 
+func TestApplication_StartAsGoRoutine(t *testing.T) {
+	factories, err := defaultcomponents.Components()
+	require.NoError(t, err)
+
+	params := Parameters{
+		ApplicationStartInfo: ApplicationStartInfo{
+			ExeName:  "otelcol",
+			LongName: "InProcess Collector",
+			Version:  version.Version,
+			GitHash:  version.GitHash,
+		},
+		ConfigFactory: func(v *viper.Viper, factories config.Factories) (*configmodels.Config, error) {
+			return constructMimumalOpConfig(t, factories), nil
+		},
+		Factories: factories,
+	}
+	app, err := New(params)
+	require.NoError(t, err)
+	app.Command().SetArgs([]string{
+		"--metrics-level=NONE",
+	})
+
+	appDone := make(chan struct{})
+	go func() {
+		defer close(appDone)
+		appErr := app.Start()
+		if appErr != nil {
+			err = appErr
+		}
+	}()
+
+	assert.Equal(t, Starting, <-app.GetStateChannel())
+	assert.Equal(t, Running, <-app.GetStateChannel())
+
+	app.SignalTestComplete()
+	app.SignalTestComplete()
+	<-appDone
+	assert.Equal(t, Closing, <-app.GetStateChannel())
+	assert.Equal(t, Closed, <-app.GetStateChannel())
+}
+
 // isAppAvailable checks if the healthcheck server at the given endpoint is
 // returning `available`.
 func isAppAvailable(t *testing.T, healthCheckEndPoint string) bool {
@@ -412,3 +454,32 @@ func TestApplication_GetExporters(t *testing.T) {
 	close(app.stopTestChan)
 	<-appDone
 }
+
+func constructMimumalOpConfig(t *testing.T, factories config.Factories) *configmodels.Config {
+	configStr := `
+receivers:
+  otlp:
+exporters:
+  logging:
+processors:
+  batch:
+
+extensions:
+
+service:
+  extensions:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [logging]
+`
+	v := viper.NewWithOptions(viper.KeyDelimiter("::"))
+	v.SetConfigType("yaml")
+	v.ReadConfig(strings.NewReader(configStr))
+	cfg, err := config.Load(v, factories)
+	assert.NoError(t, err)
+	err = config.ValidateConfig(cfg, zap.NewNop())
+	assert.NoError(t, err)
+	return cfg
+}

testbed/README.md

@@ -1,4 +1,41 @@
 # OpenTelemetry Collector Testbed
 
-Testbed is a controlled environment and tools for conducting performance tests for the Otel Collector,
-including reproducible short-term benchmarks,long-running stability tests and maximum load stress tests.
+Testbed is a controlled environment and tools for conducting end-to-end tests for the Otel Collector,
+including reproducible short-term benchmarks, correctness tests, long-running stability tests and 
+maximum load stress tests.
+
+## Usage
+
+For each type of tests that should have a summary report create a new directory and then a test suite function
+which utilizes `*testing.M`. This function should delegate all functionality to `testbed.DoTestMain` supplying
+a global instance of `testbed.TestResultsSummary` to it.
+
+Each test case within the suite should create a `testbed.TestCase` and supply implementations of each of the various
+interfaces the `NewTestCase` function takes as parameters.
+
+## Pluggable Test Components
+
+* `DataProvider` - Generates test data to send to receiver under test.
+  * `PerfTestDataProvider` - Implementation of the `DataProvider` for use in performance tests. Tracing IDs are based on the incremented batch and data items counters.
+  * `GoldenDataProvider` - Implementation of `DataProvider` for use in correctness tests. Provides data from the "Golden" dataset generated using pairwise combinatorial testing techniques.
+* `DataSender` - Sends data to the collector instance under test.
+  * `JaegerGRPCDataSender` - Implementation of `DataSender` which sends to `jaeger` receiver.
+  * `OCTraceDataSender` - Implementation of `DataSender` which sends to `opencensus` receiver.
+  * `OCMetricsDataSender` - Implementation of `DataSender` which sends to `opencensus` receiver.
+  * `OTLPTraceDataSender` - Implementation of `DataSender` which sends to `otlp` receiver.
+  * `OTLPMetricsDataSender` - Implementation of `DataSender` which sends to `otlp` receiver.
+  * `ZipkinDataSender` - Implementation of `DataSender` which sends to `zipkin` receiver.
+* `DataReceiver` - Receives data from the collector instance under test and stores it for use in test assertions.
+  * `OCDataReceiver` - Implementation of `DataReceiver` which receives data from `opencensus` exporter.
+  * `JaegerDataReceiver` - Implementation of `DataReceiver` which receives data from `jaeger` exporter.
+  * `OTLPDataReceiver` - Implementation of `DataReceiver` which receives data from `otlp` exporter.
+  * `ZipkinDataReceiver` - Implementation of `DataReceiver` which receives data from `zipkin` exporter.
+* `OtelcolRunner` - Configures, starts and stops one or more instances of otelcol which will be the subject of testing being executed.
+  * `ChildProcess` - Implementation of `OtelcolRunner` runs a single otelcol as a child process on the same machine as the test executor.
+  * `InProcessCollector` - Implementation of `OtelcolRunner` runs a single otelcol as a go routine within the same process as the test executor.
+* `TestCaseValidator` - Validates and reports on test results.
+  * `PerfTestValidator` - Implementation of `TestCaseValidator` for test suites using `PerformanceResults` for summarizing results.
+  * `CorrectnessTestValidator` - Implementation of `TestCaseValidator` for test suites using `CorrectnessResults` for summarizing results.
+* `TestResultsSummary` - Records itemized test case results plus a summary of one category of testing.
+  * `PerformanceResults` - Implementation of `TestResultsSummary` with fields suitable for reporting performance test results.
+  * `CorrectnessResults` - Implementation of `TestResultsSummary` with fields suitable for reporting data translation correctness test results.

testbed/correctness/.gitignore

@@ -0,0 +1,2 @@
+results/*
+!results/BASELINE.md