From bbc03fa6f1ad325293b5bd5c14f01a46585a5bbe Mon Sep 17 00:00:00 2001 From: Zhongyang Wu Date: Thu, 8 Oct 2020 23:19:42 -0400 Subject: [PATCH] [propagator] Add Jaeger propagator (#375) * [propagator] Initial commit of jaeger propagator. * [propagator] Add more test & copyright header. * [propagator] Add injection integration test. * [propagator] Address comments. * [propagator] Address the comment. Change traceID so that we can verify the propagator will not just take the low 64 bits. * [propagator] Add necessary doc for functions and types Co-authored-by: Tyler Yahn --- propagators/jaeger/doc.go | 17 ++ propagators/jaeger/jaeger_data_test.go | 270 ++++++++++++++++++ propagators/jaeger/jaeger_example_test.go | 30 ++ propagators/jaeger/jaeger_integration_test.go | 119 ++++++++ propagators/jaeger/jaeger_propagator.go | 160 +++++++++++ propagators/jaeger/jaeger_propagator_test.go | 163 +++++++++++ 6 files changed, 759 insertions(+) create mode 100644 propagators/jaeger/doc.go create mode 100644 propagators/jaeger/jaeger_data_test.go create mode 100644 propagators/jaeger/jaeger_example_test.go create mode 100644 propagators/jaeger/jaeger_integration_test.go create mode 100644 propagators/jaeger/jaeger_propagator.go create mode 100644 propagators/jaeger/jaeger_propagator_test.go diff --git a/propagators/jaeger/doc.go b/propagators/jaeger/doc.go new file mode 100644 index 00000000000..29f5bbd3439 --- /dev/null +++ b/propagators/jaeger/doc.go @@ -0,0 +1,17 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This package implements the Jaeger propagator specification as defined +// at https://www.jaegertracing.io/docs/1.18/client-libraries/#propagation-format +package jaeger // import "go.opentelemetry.io/contrib/propagators/jaeger" diff --git a/propagators/jaeger/jaeger_data_test.go b/propagators/jaeger/jaeger_data_test.go new file mode 100644 index 00000000000..f5f7946feaf --- /dev/null +++ b/propagators/jaeger/jaeger_data_test.go @@ -0,0 +1,270 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package jaeger_test + +import ( + "fmt" + + "go.opentelemetry.io/otel/api/trace" +) + +const ( + traceID16Str = "a3ce929d0e0e4736" + traceID32Str = "a1ce929d0e0e4736a3ce929d0e0e4736" + spanIDStr = "00f067aa0ba902b7" + jaegerHeader = "uber-trace-id" +) + +var ( + traceID16 = trace.ID{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa3, 0xce, 0x92, 0x9d, 0x0e, 0x0e, 0x47, 0x36} + traceID32 = trace.ID{0xa1, 0xce, 0x92, 0x9d, 0x0e, 0x0e, 0x47, 0x36, 0xa3, 0xce, 0x92, 0x9d, 0x0e, 0x0e, 0x47, 0x36} + spanID = trace.SpanID{0x00, 0xf0, 0x67, 0xaa, 0x0b, 0xa9, 0x02, 0xb7} +) + +type extractTest struct { + name string + headers map[string]string + expected trace.SpanContext +} + +var extractHeaders = []extractTest{ + { + "empty", + map[string]string{}, + trace.SpanContext{}, + }, + { + "sampling state not sample", + map[string]string{ + jaegerHeader: fmt.Sprintf("%s:%s:0:0", traceID32Str, spanIDStr), + }, + trace.SpanContext{ + TraceID: traceID32, + SpanID: spanID, + }, + }, + { + "sampling state sampled", + map[string]string{ + jaegerHeader: fmt.Sprintf("%s:%s:0:1", traceID32Str, spanIDStr), + }, + trace.SpanContext{ + TraceID: traceID32, + SpanID: spanID, + TraceFlags: trace.FlagsSampled, + }, + }, + { + "sampling state debug", + map[string]string{ + jaegerHeader: fmt.Sprintf("%s:%s:0:3", traceID32Str, spanIDStr), + }, + trace.SpanContext{ + TraceID: traceID32, + SpanID: spanID, + TraceFlags: trace.FlagsSampled | trace.FlagsDebug, + }, + }, + { + "sampling state debug but sampled bit didn't set, result in not sampled decision", + map[string]string{ + jaegerHeader: fmt.Sprintf("%s:%s:0:2", traceID32Str, spanIDStr), + }, + trace.SpanContext{ + TraceID: traceID32, + SpanID: spanID, + }, + }, + { + "flag can be various length", + map[string]string{ + jaegerHeader: fmt.Sprintf("%s:%s:0:00001", traceID32Str, spanIDStr), + }, + trace.SpanContext{ + TraceID: traceID32, + SpanID: spanID, + TraceFlags: trace.FlagsSampled, + }, + }, + { + "flag can be hex numbers", + map[string]string{ + jaegerHeader: fmt.Sprintf("%s:%s:0:ff", traceID32Str, spanIDStr), + }, + trace.SpanContext{ + TraceID: traceID32, + SpanID: spanID, + TraceFlags: trace.FlagsDebug | trace.FlagsSampled, + }, + }, + { + "left padding 64 bit trace ID", + map[string]string{ + jaegerHeader: fmt.Sprintf("%s:%s:0:1", traceID16Str, spanIDStr), + }, + trace.SpanContext{ + TraceID: traceID16, + SpanID: spanID, + TraceFlags: trace.FlagsSampled, + }, + }, + { + "128 bit trace ID", + map[string]string{ + jaegerHeader: fmt.Sprintf("%s:%s:0:1", traceID32Str, spanIDStr), + }, + trace.SpanContext{ + TraceID: traceID32, + SpanID: spanID, + TraceFlags: trace.FlagsSampled, + }, + }, + { + "ignore parent span id", + map[string]string{ + jaegerHeader: fmt.Sprintf("%s:%s:whatever:1", traceID32Str, spanIDStr), + }, + trace.SpanContext{ + TraceID: traceID32, + SpanID: spanID, + TraceFlags: trace.FlagsSampled, + }, + }, +} + +var invalidExtractHeaders = []extractTest{ + { + name: "trace ID length > 32", + headers: map[string]string{ + jaegerHeader: fmt.Sprintf("%s:%s:0:1", traceID32Str+"0000", spanIDStr), + }, + }, + { + name: "trace ID length is not 32 or 16", + headers: map[string]string{ + jaegerHeader: fmt.Sprintf("%s:%s:0:1", "1234567890abcd01234", spanIDStr), + }, + }, + { + name: "span ID length is not 16 or 32", + headers: map[string]string{ + jaegerHeader: fmt.Sprintf("%s:%s:0:1", traceID32Str, spanIDStr+"0000"), + }, + }, + { + name: "invalid trace ID", + headers: map[string]string{ + jaegerHeader: fmt.Sprintf("%s:%s:0:1", "zcd00v0000000000a3ce929d0e0e4736", spanIDStr), + }, + }, + { + name: "invalid span ID", + headers: map[string]string{ + jaegerHeader: fmt.Sprintf("%s:%s:0:1", traceID32Str, "00f0wiredba902b7"), + }, + }, + { + name: "invalid flags", + headers: map[string]string{ + jaegerHeader: fmt.Sprintf("%s:%s:0:wired", traceID32Str, spanIDStr), + }, + }, + { + name: "invalid separator", + headers: map[string]string{ + jaegerHeader: fmt.Sprintf("%s-%s-0-1", traceID32Str, spanIDStr), + }, + }, + { + name: "missing jaeger header", + headers: map[string]string{ + jaegerHeader + "not": fmt.Sprintf("%s:%s:0:1", traceID32Str, spanIDStr), + }, + }, + { + name: "empty header value", + headers: map[string]string{ + jaegerHeader: "", + }, + }, +} + +type injectTest struct { + name string + sc trace.SpanContext + wantHeaders map[string]string +} + +var injectHeaders = []injectTest{ + { + name: "sampled", + sc: trace.SpanContext{ + TraceID: traceID32, + SpanID: spanID, + TraceFlags: trace.FlagsSampled, + }, + wantHeaders: map[string]string{ + jaegerHeader: fmt.Sprintf("%s:%s:0:1", traceID32Str, spanIDStr), + }, + }, + { + name: "debug", + sc: trace.SpanContext{ + TraceID: traceID32, + SpanID: spanID, + TraceFlags: trace.FlagsSampled | trace.FlagsDebug, + }, + wantHeaders: map[string]string{ + jaegerHeader: fmt.Sprintf("%s:%s:0:3", traceID32Str, spanIDStr), + }, + }, + { + name: "not sampled", + sc: trace.SpanContext{ + TraceID: traceID32, + SpanID: spanID, + }, + wantHeaders: map[string]string{ + jaegerHeader: fmt.Sprintf("%s:%s:0:0", traceID32Str, spanIDStr), + }, + }, +} + +var invalidInjectHeaders = []injectTest{ + { + name: "empty", + sc: trace.SpanContext{}, + }, + { + name: "missing traceID", + sc: trace.SpanContext{ + SpanID: spanID, + TraceFlags: trace.FlagsSampled, + }, + }, + { + name: "missing spanID", + sc: trace.SpanContext{ + TraceID: traceID32, + TraceFlags: trace.FlagsSampled, + }, + }, + { + name: "missing both traceID and spanID", + sc: trace.SpanContext{ + TraceFlags: trace.FlagsSampled, + }, + }, +} diff --git a/propagators/jaeger/jaeger_example_test.go b/propagators/jaeger/jaeger_example_test.go new file mode 100644 index 00000000000..b67b23721ba --- /dev/null +++ b/propagators/jaeger/jaeger_example_test.go @@ -0,0 +1,30 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package jaeger_test + +import ( + "go.opentelemetry.io/contrib/propagators/jaeger" + "go.opentelemetry.io/otel/api/global" + "go.opentelemetry.io/otel/api/propagation" +) + +func ExampleJaeger() { + jaeger := jaeger.Jaeger{} + // register jaeger propagator + global.SetPropagators(propagation.New( + propagation.WithExtractors(jaeger), + propagation.WithInjectors(jaeger), + )) +} diff --git a/propagators/jaeger/jaeger_integration_test.go b/propagators/jaeger/jaeger_integration_test.go new file mode 100644 index 00000000000..cb55d15a8cf --- /dev/null +++ b/propagators/jaeger/jaeger_integration_test.go @@ -0,0 +1,119 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package jaeger_test + +import ( + "context" + "net/http" + "testing" + + "github.com/google/go-cmp/cmp" + + mocktracer "go.opentelemetry.io/contrib/internal/trace" + "go.opentelemetry.io/contrib/propagators/jaeger" + "go.opentelemetry.io/otel/api/propagation" + "go.opentelemetry.io/otel/api/trace" +) + +var ( + mockTracer = mocktracer.NewTracer("") + _, mockSpan = mockTracer.Start(context.Background(), "") +) + +func TestExtractJaeger(t *testing.T) { + testGroup := []struct { + name string + testcases []extractTest + }{ + { + name: "valid test case", + testcases: extractHeaders, + }, + { + name: "invalid test case", + testcases: invalidExtractHeaders, + }, + } + + for _, tg := range testGroup { + propagator := jaeger.Jaeger{} + props := propagation.New(propagation.WithExtractors(propagator)) + + for _, tc := range tg.testcases { + t.Run(tc.name, func(t *testing.T) { + req, _ := http.NewRequest("GET", "http://example.com", nil) + for k, v := range tc.headers { + req.Header.Set(k, v) + } + + ctx := context.Background() + ctx = propagation.ExtractHTTP(ctx, props, req.Header) + resSc := trace.RemoteSpanContextFromContext(ctx) + if diff := cmp.Diff(resSc, tc.expected); diff != "" { + t.Errorf("%s: %s: -got +want %s", tg.name, tc.name, diff) + } + }) + } + } +} + +type testSpan struct { + trace.Span + sc trace.SpanContext +} + +func (s testSpan) SpanContext() trace.SpanContext { + return s.sc +} + +func TestInjectJaeger(t *testing.T) { + testGroup := []struct { + name string + testcases []injectTest + }{ + { + name: "valid test case", + testcases: injectHeaders, + }, + { + name: "invalid test case", + testcases: invalidInjectHeaders, + }, + } + + for _, tg := range testGroup { + for _, tc := range tg.testcases { + propagator := jaeger.Jaeger{} + t.Run(tc.name, func(t *testing.T) { + req, _ := http.NewRequest("GET", "http://example.com", nil) + ctx := trace.ContextWithSpan( + context.Background(), + testSpan{ + Span: mockSpan, + sc: tc.sc, + }, + ) + propagator.Inject(ctx, req.Header) + + for h, v := range tc.wantHeaders { + result, want := req.Header.Get(h), v + if diff := cmp.Diff(result, want); diff != "" { + t.Errorf("%s: %s, header=%s: -got +want %s", tg.name, tc.name, h, diff) + } + } + }) + } + } +} diff --git a/propagators/jaeger/jaeger_propagator.go b/propagators/jaeger/jaeger_propagator.go new file mode 100644 index 00000000000..0291526f430 --- /dev/null +++ b/propagators/jaeger/jaeger_propagator.go @@ -0,0 +1,160 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package jaeger + +import ( + "context" + "errors" + "fmt" + "strconv" + "strings" + + "go.opentelemetry.io/otel/api/propagation" + "go.opentelemetry.io/otel/api/trace" +) + +const ( + jaegerHeader = "uber-trace-id" + separator = ":" + traceID64bitsWidth = 64 / 4 + traceID128bitsWidth = 128 / 4 + spanIDWidth = 64 / 4 + + traceIDPadding = "0000000000000000" + + flagsDebug = 0x02 + flagsSampled = 0x01 + flagsNotSampled = 0x00 + + deprecatedParentSpanID = "0" +) + +var ( + empty = trace.EmptySpanContext() + + errMalformedTraceContextVal = errors.New("header value of uber-trace-id should contain four different part separated by : ") + errInvalidTraceIDLength = errors.New("invalid trace id length, must be either 16 or 32") + errMalformedTraceID = errors.New("cannot decode trace id from header, should be a string of hex, lowercase trace id can't be all zero") + errInvalidSpanIDLength = errors.New("invalid span id length, must be 16") + errMalformedSpanID = errors.New("cannot decode span id from header, should be a string of hex, lowercase span id can't be all zero") + errMalformedFlag = errors.New("cannot decode flag") +) + +// Jaeger propagator serializes SpanContext to/from Jaeger Headers +// +// Jaeger format: +// +// uber-trace-id: {trace-id}:{span-id}:{parent-span-id}:{flags} +type Jaeger struct{} + +var _ propagation.HTTPPropagator = &Jaeger{} + +// Inject injects a context to the supplier following jaeger format. +// The parent span ID is set to an dummy parent span id as the most implementations do. +func (jaeger Jaeger) Inject(ctx context.Context, supplier propagation.HTTPSupplier) { + sc := trace.SpanFromContext(ctx).SpanContext() + headers := []string{} + if !sc.TraceID.IsValid() || !sc.SpanID.IsValid() { + return + } + headers = append(headers, sc.TraceID.String(), sc.SpanID.String(), deprecatedParentSpanID) + if sc.IsDebug() { + headers = append(headers, fmt.Sprintf("%x", flagsDebug|flagsSampled)) + } else if sc.IsSampled() { + headers = append(headers, fmt.Sprintf("%x", flagsSampled)) + } else { + headers = append(headers, fmt.Sprintf("%x", flagsNotSampled)) + } + + supplier.Set(jaegerHeader, strings.Join(headers, separator)) +} + +// Extract extracts a context from the supplier if it contains Jaeger headers. +func (jaeger Jaeger) Extract(ctx context.Context, supplier propagation.HTTPSupplier) context.Context { + // extract tracing information + if h := supplier.Get(jaegerHeader); h != "" { + sc, err := extract(h) + if err == nil && sc.IsValid() { + return trace.ContextWithRemoteSpanContext(ctx, sc) + } + } + + return ctx +} + +func extract(headerVal string) (trace.SpanContext, error) { + var ( + sc = trace.SpanContext{} + err error + ) + + parts := strings.Split(headerVal, separator) + if len(parts) != 4 { + return empty, errMalformedTraceContextVal + } + + // extract trace ID + if parts[0] != "" { + id := parts[0] + if len(id) != traceID128bitsWidth && len(id) != traceID64bitsWidth { + return empty, errInvalidTraceIDLength + } + // padding when length is 16 + if len(id) == traceID64bitsWidth { + id = traceIDPadding + id + } + sc.TraceID, err = trace.IDFromHex(id) + if err != nil { + return empty, errMalformedTraceID + } + } + + // extract span ID + if parts[1] != "" { + id := parts[1] + if len(id) != spanIDWidth { + return empty, errInvalidSpanIDLength + } + sc.SpanID, err = trace.SpanIDFromHex(id) + if err != nil { + return empty, errMalformedSpanID + } + } + + // skip third part as it is deprecated + + // extract flag + if parts[3] != "" { + flagStr := parts[3] + flag, err := strconv.ParseInt(flagStr, 16, 64) + if err != nil { + return empty, errMalformedFlag + } + if flag&flagsSampled == flagsSampled { + // if sample bit is set, we check if debug bit is also set + if flag&flagsDebug == flagsDebug { + sc.TraceFlags |= trace.FlagsSampled | trace.FlagsDebug + } else { + sc.TraceFlags |= trace.FlagsSampled + } + } + // ignore other bit, including firehose since we don't have corresponding flag in trace context. + } + return sc, nil +} + +func (jaeger Jaeger) GetAllKeys() []string { + return []string{jaegerHeader} +} diff --git a/propagators/jaeger/jaeger_propagator_test.go b/propagators/jaeger/jaeger_propagator_test.go new file mode 100644 index 00000000000..7bbcc74ab07 --- /dev/null +++ b/propagators/jaeger/jaeger_propagator_test.go @@ -0,0 +1,163 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package jaeger + +import ( + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + + "go.opentelemetry.io/otel/api/trace" +) + +var ( + traceID = trace.ID{0, 0, 0, 0, 0, 0, 0, 0, 0x7b, 0, 0, 0, 0, 0, 0x1, 0xc8} + traceID128Str = "00000000000000007b000000000001c8" + zeroTraceIDStr = "00000000000000000000000000000000" + traceID64Str = "7b000000000001c8" + spanID = trace.SpanID{0, 0, 0, 0, 0, 0, 0, 0x7b} + zeroSpanIDStr = "0000000000000000" + spanIDStr = "000000000000007b" +) + +func TestJaeger_Extract(t *testing.T) { + testData := []struct { + traceID string + spanID string + parentSpanID string + flags string + expected trace.SpanContext + err error + }{ + { + traceID128Str, spanIDStr, deprecatedParentSpanID, "1", + trace.SpanContext{ + TraceID: traceID, + SpanID: spanID, + TraceFlags: trace.FlagsSampled, + }, + nil, + }, + { + traceID64Str, spanIDStr, deprecatedParentSpanID, "1", + trace.SpanContext{ + TraceID: traceID, + SpanID: spanID, + TraceFlags: trace.FlagsSampled, + }, + nil, + }, + { + traceID128Str, spanIDStr, deprecatedParentSpanID, "3", + trace.SpanContext{ + TraceID: traceID, + SpanID: spanID, + TraceFlags: trace.FlagsSampled | trace.FlagsDebug, + }, + nil, + }, + { + // if we didn't set sampled bit when debug bit is 1, then assuming it's not sampled + traceID128Str, spanIDStr, deprecatedParentSpanID, "2", + trace.SpanContext{ + TraceID: traceID, + SpanID: spanID, + TraceFlags: 0x00, + }, + nil, + }, + { + // ignore firehose bit since we don't really have this feature in otel span context + traceID128Str, spanIDStr, deprecatedParentSpanID, "8", + trace.SpanContext{ + TraceID: traceID, + SpanID: spanID, + TraceFlags: 0x00, + }, + nil, + }, + { + traceID128Str, spanIDStr, deprecatedParentSpanID, "9", + trace.SpanContext{ + TraceID: traceID, + SpanID: spanID, + TraceFlags: trace.FlagsSampled, + }, + nil, + }, + { + traceID128Str, spanIDStr, "wired stuff", "1", + trace.SpanContext{ + TraceID: traceID, + SpanID: spanID, + TraceFlags: trace.FlagsSampled, + }, + nil, + }, + { + fmt.Sprintf("%32s", "This_is_a_string_len_64"), spanIDStr, deprecatedParentSpanID, "1", + trace.SpanContext{}, + errMalformedTraceID, + }, + { + "000000000007b00000000000001c8", spanIDStr, deprecatedParentSpanID, "1", + trace.SpanContext{}, + errInvalidTraceIDLength, + }, + { + traceID128Str, fmt.Sprintf("%16s", "wiredspanid"), deprecatedParentSpanID, "1", + trace.SpanContext{}, + errMalformedSpanID, + }, + { + traceID128Str, "0000000000010", deprecatedParentSpanID, "1", + trace.SpanContext{}, + errInvalidSpanIDLength, + }, + { + // reject invalid traceID(0) and spanID(0) + zeroTraceIDStr, zeroSpanIDStr, deprecatedParentSpanID, "1", + trace.SpanContext{}, + errMalformedTraceID, + }, + { + // reject invalid traceID(0) and spanID(0) + traceID128Str, zeroSpanIDStr, deprecatedParentSpanID, "1", + trace.SpanContext{}, + errMalformedSpanID, + }, + } + + for _, test := range testData { + headerVal := strings.Join([]string{test.traceID, test.spanID, test.parentSpanID, test.flags}, separator) + sc, err := extract(headerVal) + + info := []interface{}{ + "trace ID: %q, span ID: %q, parent span ID: %q, sampled: %q, flags: %q", + test.traceID, + test.spanID, + test.parentSpanID, + test.flags, + } + + if !assert.Equal(t, test.err, err, info...) { + continue + } + + assert.Equal(t, test.expected, sc, info...) + } +}