From 6ad7d673dbcb9ef99c3bad7b4731522a6ddf1ad5 Mon Sep 17 00:00:00 2001 From: Bogdan Drutu Date: Tue, 2 Mar 2021 14:06:57 -0800 Subject: [PATCH] Change goldendataset trace to return pdata (#2563) Signed-off-by: Bogdan Drutu --- internal/goldendataset/traces_generator.go | 23 ++++----- .../goldendataset/traces_generator_test.go | 7 +-- .../correctness/traces/correctness_test.go | 3 +- testbed/testbed/data_providers.go | 47 +++++++++---------- testbed/testbed/data_providers_test.go | 2 +- .../internaldata/resource_to_oc_test.go | 7 +-- translator/internaldata/traces_to_oc_test.go | 11 ++--- .../jaeger/traces_to_jaegerproto_test.go | 13 ++--- .../trace/zipkin/traces_to_zipkinv2_test.go | 13 ++--- 9 files changed, 47 insertions(+), 79 deletions(-) diff --git a/internal/goldendataset/traces_generator.go b/internal/goldendataset/traces_generator.go index 8a614e82d80..df1d67a6cc4 100644 --- a/internal/goldendataset/traces_generator.go +++ b/internal/goldendataset/traces_generator.go @@ -17,24 +17,25 @@ package goldendataset import ( "fmt" "io" + "math/rand" + "go.opentelemetry.io/collector/consumer/pdata" otlpcommon "go.opentelemetry.io/collector/internal/data/protogen/common/v1" otlptrace "go.opentelemetry.io/collector/internal/data/protogen/trace/v1" ) -// GenerateResourceSpans generates a slice of OTLP ResourceSpans objects based on the PICT-generated pairwise +// GenerateTraces generates a slice of OTLP ResourceSpans objects based on the PICT-generated pairwise // parameters defined in the parameters file specified by the tracePairsFile parameter. The pairs to generate -// spans for for defined in the file specified by the spanPairsFile parameter. The random parameter injects the -// random number generator to use in generating IDs and other random values. +// spans for for defined in the file specified by the spanPairsFile parameter. // The slice of ResourceSpans are returned. If an err is returned, the slice elements will be nil. -func GenerateResourceSpans(tracePairsFile string, spanPairsFile string, - random io.Reader) ([]*otlptrace.ResourceSpans, error) { +func GenerateTraces(tracePairsFile string, spanPairsFile string) ([]pdata.Traces, error) { + random := io.Reader(rand.New(rand.NewSource(42))) pairsData, err := loadPictOutputFile(tracePairsFile) if err != nil { return nil, err } pairsTotal := len(pairsData) - 1 - spans := make([]*otlptrace.ResourceSpans, pairsTotal) + traces := make([]pdata.Traces, pairsTotal) for index, values := range pairsData { if index == 0 { continue @@ -44,22 +45,22 @@ func GenerateResourceSpans(tracePairsFile string, spanPairsFile string, InstrumentationLibrary: PICTInputInstrumentationLibrary(values[TracesColumnInstrumentationLibrary]), Spans: PICTInputSpans(values[TracesColumnSpans]), } - rscSpan, spanErr := GenerateResourceSpan(tracingInputs, spanPairsFile, random) + rscSpan, spanErr := generateResourceSpan(tracingInputs, spanPairsFile, random) if spanErr != nil { err = spanErr } - spans[index-1] = rscSpan + traces[index-1] = pdata.TracesFromOtlp([]*otlptrace.ResourceSpans{rscSpan}) } - return spans, err + return traces, err } -// GenerateResourceSpan generates a single OTLP ResourceSpans populated based on the provided inputs. They are: +// generateResourceSpan generates a single OTLP ResourceSpans populated based on the provided inputs. They are: // tracingInputs - the pairwise combination of field value variations for this ResourceSpans // spanPairsFile - the file with the PICT-generated parameter combinations to generate spans for // random - the random number generator to use in generating ID values // // The generated resource spans. If err is not nil, some or all of the resource spans fields will be nil. -func GenerateResourceSpan(tracingInputs *PICTTracingInputs, spanPairsFile string, +func generateResourceSpan(tracingInputs *PICTTracingInputs, spanPairsFile string, random io.Reader) (*otlptrace.ResourceSpans, error) { libSpans, err := generateLibrarySpansArray(tracingInputs, spanPairsFile, random) return &otlptrace.ResourceSpans{ diff --git a/internal/goldendataset/traces_generator_test.go b/internal/goldendataset/traces_generator_test.go index dbb8a009999..10a1cfd097b 100644 --- a/internal/goldendataset/traces_generator_test.go +++ b/internal/goldendataset/traces_generator_test.go @@ -15,17 +15,14 @@ package goldendataset import ( - "io" - "math/rand" "testing" "github.com/stretchr/testify/assert" ) func TestGenerateTraces(t *testing.T) { - random := io.Reader(rand.New(rand.NewSource(42))) - rscSpans, err := GenerateResourceSpans("testdata/generated_pict_pairs_traces.txt", - "testdata/generated_pict_pairs_spans.txt", random) + rscSpans, err := GenerateTraces("testdata/generated_pict_pairs_traces.txt", + "testdata/generated_pict_pairs_spans.txt") assert.Nil(t, err) assert.Equal(t, 32, len(rscSpans)) } diff --git a/testbed/correctness/traces/correctness_test.go b/testbed/correctness/traces/correctness_test.go index 6d712dd7d45..1b57bf7aba0 100644 --- a/testbed/correctness/traces/correctness_test.go +++ b/testbed/correctness/traces/correctness_test.go @@ -61,8 +61,7 @@ func testWithTracingGoldenDataset( dataProvider := testbed.NewGoldenDataProvider( "../../../internal/goldendataset/testdata/generated_pict_pairs_traces.txt", "../../../internal/goldendataset/testdata/generated_pict_pairs_spans.txt", - "", - 161803) + "") factories, err := defaultcomponents.Components() require.NoError(t, err, "default components resulted in: %v", err) runner := testbed.NewInProcessCollector(factories) diff --git a/testbed/testbed/data_providers.go b/testbed/testbed/data_providers.go index cc5f4324cfd..58257cf9944 100644 --- a/testbed/testbed/data_providers.go +++ b/testbed/testbed/data_providers.go @@ -17,9 +17,7 @@ package testbed import ( "encoding/binary" "fmt" - "io" "log" - "math/rand" "os" "strconv" "time" @@ -216,12 +214,12 @@ func (dp *PerfTestDataProvider) GenerateLogs() (pdata.Logs, bool) { type GoldenDataProvider struct { tracePairsFile string spanPairsFile string - random io.Reader batchesGenerated *atomic.Uint64 dataItemsGenerated *atomic.Uint64 - resourceSpans []*otlptrace.ResourceSpans - spansIndex int - spansMap map[string]*otlptrace.Span + + tracesGenerated []pdata.Traces + tracesIndex int + spansMap map[string]*otlptrace.Span metricPairsFile string metricsGenerated []pdata.Metrics @@ -230,13 +228,11 @@ type GoldenDataProvider struct { // NewGoldenDataProvider creates a new instance of GoldenDataProvider which generates test data based // on the pairwise combinations specified in the tracePairsFile and spanPairsFile input variables. -// The supplied randomSeed is used to initialize the random number generator used in generating tracing IDs. -func NewGoldenDataProvider(tracePairsFile string, spanPairsFile string, metricPairsFile string, randomSeed int64) *GoldenDataProvider { +func NewGoldenDataProvider(tracePairsFile string, spanPairsFile string, metricPairsFile string) *GoldenDataProvider { return &GoldenDataProvider{ tracePairsFile: tracePairsFile, spanPairsFile: spanPairsFile, metricPairsFile: metricPairsFile, - random: io.Reader(rand.New(rand.NewSource(randomSeed))), } } @@ -246,27 +242,22 @@ func (dp *GoldenDataProvider) SetLoadGeneratorCounters(batchesGenerated *atomic. } func (dp *GoldenDataProvider) GenerateTraces() (pdata.Traces, bool) { - if dp.resourceSpans == nil { + if dp.tracesGenerated == nil { var err error - dp.resourceSpans, err = goldendataset.GenerateResourceSpans(dp.tracePairsFile, dp.spanPairsFile, dp.random) + dp.tracesGenerated, err = goldendataset.GenerateTraces(dp.tracePairsFile, dp.spanPairsFile) if err != nil { log.Printf("cannot generate traces: %s", err) - dp.resourceSpans = make([]*otlptrace.ResourceSpans, 0) + dp.tracesGenerated = nil } } dp.batchesGenerated.Inc() - if dp.spansIndex >= len(dp.resourceSpans) { - return pdata.TracesFromOtlp(make([]*otlptrace.ResourceSpans, 0)), true - } - resourceSpans := make([]*otlptrace.ResourceSpans, 1) - resourceSpans[0] = dp.resourceSpans[dp.spansIndex] - dp.spansIndex++ - spanCount := uint64(0) - for _, libSpans := range resourceSpans[0].InstrumentationLibrarySpans { - spanCount += uint64(len(libSpans.Spans)) + if dp.tracesIndex >= len(dp.tracesGenerated) { + return pdata.NewTraces(), true } - dp.dataItemsGenerated.Add(spanCount) - return pdata.TracesFromOtlp(resourceSpans), false + td := dp.tracesGenerated[dp.tracesIndex] + dp.tracesIndex++ + dp.dataItemsGenerated.Add(uint64(td.SpanCount())) + return td, false } func (dp *GoldenDataProvider) GenerateMetrics() (pdata.Metrics, bool) { @@ -277,8 +268,8 @@ func (dp *GoldenDataProvider) GenerateMetrics() (pdata.Metrics, bool) { log.Printf("cannot generate metrics: %s", err) } } - numMetricsGenerated := len(dp.metricsGenerated) - if dp.metricsIndex == numMetricsGenerated { + dp.batchesGenerated.Inc() + if dp.metricsIndex == len(dp.metricsGenerated) { return pdata.Metrics{}, true } pdm := dp.metricsGenerated[dp.metricsIndex] @@ -294,7 +285,11 @@ func (dp *GoldenDataProvider) GenerateLogs() (pdata.Logs, bool) { func (dp *GoldenDataProvider) GetGeneratedSpan(traceID pdata.TraceID, spanID pdata.SpanID) *otlptrace.Span { if dp.spansMap == nil { - dp.spansMap = populateSpansMap(dp.resourceSpans) + var resourceSpansList []*otlptrace.ResourceSpans + for _, td := range dp.tracesGenerated { + resourceSpansList = append(resourceSpansList, pdata.TracesToOtlp(td)...) + } + dp.spansMap = populateSpansMap(resourceSpansList) } key := traceIDAndSpanIDToString(traceID, spanID) return dp.spansMap[key] diff --git a/testbed/testbed/data_providers_test.go b/testbed/testbed/data_providers_test.go index f80adae298c..7b001c1d73a 100644 --- a/testbed/testbed/data_providers_test.go +++ b/testbed/testbed/data_providers_test.go @@ -26,7 +26,7 @@ import ( const metricsPictPairsFile = "../../internal/goldendataset/testdata/generated_pict_pairs_metrics.txt" func TestGoldenDataProvider(t *testing.T) { - dp := NewGoldenDataProvider("", "", metricsPictPairsFile, 42) + dp := NewGoldenDataProvider("", "", metricsPictPairsFile) dp.SetLoadGeneratorCounters(atomic.NewUint64(0), atomic.NewUint64(0)) var ms []pdata.Metrics for { diff --git a/translator/internaldata/resource_to_oc_test.go b/translator/internaldata/resource_to_oc_test.go index 831eea6df31..78d004decc4 100644 --- a/translator/internaldata/resource_to_oc_test.go +++ b/translator/internaldata/resource_to_oc_test.go @@ -231,12 +231,7 @@ func TestResourceToOCAndBack(t *testing.T) { } for _, test := range tests { t.Run(string(test), func(t *testing.T) { - rSpans := make([]*otlptrace.ResourceSpans, 1) - rSpans[0] = &otlptrace.ResourceSpans{ - Resource: goldendataset.GenerateResource(test), - InstrumentationLibrarySpans: nil, - } - traces := pdata.TracesFromOtlp(rSpans) + traces := pdata.TracesFromOtlp([]*otlptrace.ResourceSpans{{Resource: goldendataset.GenerateResource(test)}}) expected := traces.ResourceSpans().At(0).Resource() ocNode, ocResource := internalResourceToOC(expected) actual := pdata.NewResource() diff --git a/translator/internaldata/traces_to_oc_test.go b/translator/internaldata/traces_to_oc_test.go index ffae4ed5fe0..23ed5019d86 100644 --- a/translator/internaldata/traces_to_oc_test.go +++ b/translator/internaldata/traces_to_oc_test.go @@ -15,8 +15,6 @@ package internaldata import ( - "io" - "math/rand" "testing" occommon "github.com/census-instrumentation/opencensus-proto/gen-go/agent/common/v1" @@ -28,7 +26,6 @@ import ( "google.golang.org/protobuf/types/known/wrapperspb" "go.opentelemetry.io/collector/consumer/pdata" - otlptrace "go.opentelemetry.io/collector/internal/data/protogen/trace/v1" "go.opentelemetry.io/collector/internal/goldendataset" "go.opentelemetry.io/collector/internal/testdata" "go.opentelemetry.io/collector/translator/conventions" @@ -357,13 +354,11 @@ func TestInternalToOC(t *testing.T) { } func TestInternalTracesToOCTracesAndBack(t *testing.T) { - rscSpans, err := goldendataset.GenerateResourceSpans( + tds, err := goldendataset.GenerateTraces( "../../internal/goldendataset/testdata/generated_pict_pairs_traces.txt", - "../../internal/goldendataset/testdata/generated_pict_pairs_spans.txt", - io.Reader(rand.New(rand.NewSource(2004)))) + "../../internal/goldendataset/testdata/generated_pict_pairs_spans.txt") assert.NoError(t, err) - for _, rs := range rscSpans { - td := pdata.TracesFromOtlp([]*otlptrace.ResourceSpans{rs}) + for _, td := range tds { ocNode, ocResource, ocSpans := ResourceSpansToOC(td.ResourceSpans().At(0)) assert.Equal(t, td.SpanCount(), len(ocSpans)) tdFromOC := OCToTraces(ocNode, ocResource, ocSpans) diff --git a/translator/trace/jaeger/traces_to_jaegerproto_test.go b/translator/trace/jaeger/traces_to_jaegerproto_test.go index b9dc0ecbd5e..94309e9b753 100644 --- a/translator/trace/jaeger/traces_to_jaegerproto_test.go +++ b/translator/trace/jaeger/traces_to_jaegerproto_test.go @@ -15,8 +15,6 @@ package jaeger import ( - "io" - "math/rand" "testing" "github.com/jaegertracing/jaeger/model" @@ -24,7 +22,6 @@ import ( "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/consumer/pdata" - otlptrace "go.opentelemetry.io/collector/internal/data/protogen/trace/v1" "go.opentelemetry.io/collector/internal/goldendataset" "go.opentelemetry.io/collector/internal/testdata" "go.opentelemetry.io/collector/translator/conventions" @@ -323,15 +320,11 @@ func TestInternalTracesToJaegerProto(t *testing.T) { } func TestInternalTracesToJaegerProtoBatchesAndBack(t *testing.T) { - rscSpans, err := goldendataset.GenerateResourceSpans( + tds, err := goldendataset.GenerateTraces( "../../../internal/goldendataset/testdata/generated_pict_pairs_traces.txt", - "../../../internal/goldendataset/testdata/generated_pict_pairs_spans.txt", - io.Reader(rand.New(rand.NewSource(2004)))) + "../../../internal/goldendataset/testdata/generated_pict_pairs_spans.txt") assert.NoError(t, err) - for _, rs := range rscSpans { - orig := make([]*otlptrace.ResourceSpans, 1) - orig[0] = rs - td := pdata.TracesFromOtlp(orig) + for _, td := range tds { protoBatches, err := InternalTracesToJaegerProto(td) assert.NoError(t, err) tdFromPB := ProtoBatchesToInternalTraces(protoBatches) diff --git a/translator/trace/zipkin/traces_to_zipkinv2_test.go b/translator/trace/zipkin/traces_to_zipkinv2_test.go index ac5572c1ddb..746b808fb5e 100644 --- a/translator/trace/zipkin/traces_to_zipkinv2_test.go +++ b/translator/trace/zipkin/traces_to_zipkinv2_test.go @@ -16,15 +16,12 @@ package zipkin import ( "errors" - "io" - "math/rand" "testing" zipkinmodel "github.com/openzipkin/zipkin-go/model" "github.com/stretchr/testify/assert" "go.opentelemetry.io/collector/consumer/pdata" - otlptrace "go.opentelemetry.io/collector/internal/data/protogen/trace/v1" "go.opentelemetry.io/collector/internal/goldendataset" "go.opentelemetry.io/collector/internal/testdata" ) @@ -87,15 +84,11 @@ func TestInternalTracesToZipkinSpans(t *testing.T) { } func TestInternalTracesToZipkinSpansAndBack(t *testing.T) { - rscSpans, err := goldendataset.GenerateResourceSpans( + tds, err := goldendataset.GenerateTraces( "../../../internal/goldendataset/testdata/generated_pict_pairs_traces.txt", - "../../../internal/goldendataset/testdata/generated_pict_pairs_spans.txt", - io.Reader(rand.New(rand.NewSource(2004)))) + "../../../internal/goldendataset/testdata/generated_pict_pairs_spans.txt") assert.NoError(t, err) - for _, rs := range rscSpans { - orig := make([]*otlptrace.ResourceSpans, 1) - orig[0] = rs - td := pdata.TracesFromOtlp(orig) + for _, td := range tds { zipkinSpans, err := InternalTracesToZipkinSpans(td) assert.NoError(t, err) assert.Equal(t, td.SpanCount(), len(zipkinSpans))