Add OT trace information on outgoing HTTP reqs (#685)

* Inject our trace span data into outgoing HTTP requests (via OT)

This should help us tie together spans that are generated by proxies
such as envoy (which hopefully understand the span data and pick it up
/ report it).

* Encode OT HTTP headers in envoy format

This is a pretty radical departure from how we did it
before (implicitly, via the textual map interface); now, we set the
headers in a way that one of the most popular external tools can
understand, and maybe stand a chance to get fully fleshed-out traces
as a result (:

* Propagate the "sampled" header also

Envoy needs to see it in order to pick up the span and properly
transmit it over.

* Thanks, me!

CHANGELOG.md

@@ -1,5 +1,9 @@
 # 12.0.0, in progress
 
+## Added
+
+* The OpenTracing implementation's `Tracer.Inject` in the `trace` package now sets HTTP headers in a way that tools like [Envoy](https://www.envoyproxy.io/) can propagate on traces. Thanks, [antifuchs](https://github.com/antifuchs)!
+
 ## Bugfixes
 * The signalfx client no longer reports a timeout when submission to the datapoint API endpoint encounters an error. Thanks, [antifuchs](https://github.com/antifuchs)!
 

http/http.go

@@ -131,6 +131,9 @@ func (tripper *TraceRoundTripper) RoundTrip(req *http.Request) (*http.Response,
 	span.SetTag("action", tripper.prefix)
 	defer span.ClientFinish(tripper.tc)
 
+	// Add OpenTracing headers to the request, so downstream reqs can be identified:
+	trace.GlobalTracer.InjectRequest(span.Trace, req)
+
 	hct := newHTTPClientTracer(span.Attach(req.Context()), tripper.tc, tripper.prefix)
 	req = req.WithContext(httptrace.WithClientTrace(req.Context(), hct.getClientTrace()))
 	defer hct.finishSpan()

trace/opentracing.go

@@ -20,9 +20,10 @@ import (
 
 // Lists the names of headers that a specification uses for representing trace information.
 type HeaderGroup struct {
-	TraceID   string
-	SpanID    string
-	numFormat headerGroupNumFormat
+	TraceID         string
+	SpanID          string
+	numFormat       headerGroupNumFormat
+	OutgoingHeaders map[string]string
 }
 
 type headerGroupNumFormat int
@@ -42,6 +43,9 @@ var HeaderFormats = []HeaderGroup{
 		TraceID:   "ot-tracer-traceid",
 		SpanID:    "ot-tracer-spanid",
 		numFormat: hexadecimal,
+		OutgoingHeaders: map[string]string{
+			"ot-tracer-sampled": "true",
+		},
 	},
 	// OpenTracing format.
 	HeaderGroup{
@@ -60,6 +64,10 @@ var HeaderFormats = []HeaderGroup{
 	},
 }
 
+// defaultHeaderFormat is the way .Inject sets HTTP headers by
+// default.
+var defaultHeaderFormat = HeaderFormats[0]
+
 // GlobalTracer is the… global tracer!
 var GlobalTracer = Tracer{}
 
@@ -514,7 +522,6 @@ func (tracer Tracer) ExtractRequestChild(resource string, req *http.Request, nam
 // Inject injects the provided SpanContext into the carrier for propagation.
 // It will return opentracing.ErrUnsupportedFormat if the format is not supported.
 // TODO support other SpanContext implementations
-// TODO support all the BuiltinFormats
 func (t Tracer) Inject(sm opentracing.SpanContext, format interface{}, carrier interface{}) (err error) {
 	defer func() {
 		if r := recover(); r != nil {
@@ -528,7 +535,8 @@ func (t Tracer) Inject(sm opentracing.SpanContext, format interface{}, carrier i
 		return ErrUnsupportedSpanContext
 	}
 
-	if format == opentracing.Binary {
+	switch format {
+	case opentracing.Binary:
 		// carrier is guaranteed to be an io.Writer by contract
 		w := carrier.(io.Writer)
 
@@ -541,6 +549,20 @@ func (t Tracer) Inject(sm opentracing.SpanContext, format interface{}, carrier i
 		}
 
 		return trace.ProtoMarshalTo(w)
+	case opentracing.HTTPHeaders:
+		h := carrier.(opentracing.HTTPHeadersCarrier)
+		base := 10
+		if defaultHeaderFormat.numFormat == hexadecimal {
+			base = 16
+		}
+		h.Set(defaultHeaderFormat.SpanID, strconv.FormatInt(sc.SpanID(), base))
+		h.Set(defaultHeaderFormat.TraceID, strconv.FormatInt(sc.TraceID(), base))
+		if defaultHeaderFormat.OutgoingHeaders != nil {
+			for name, value := range defaultHeaderFormat.OutgoingHeaders {
+				h.Set(name, value)
+			}
+		}
+		return nil
 	}
 
 	// If the carrier is a TextMapWriter, treat it as one, regardless of what the format is

trace/opentracing_test.go

@@ -222,6 +222,11 @@ func TestTracerInjectExtractHeader(t *testing.T) {
 	err = tracer.Inject(trace.context(), opentracing.HTTPHeaders, carrier)
 	assert.NoError(t, err)
 
+	assert.NotEqual(t, "", req.Header.Get(defaultHeaderFormat.SpanID),
+		"Headers %v should contain %q", req.Header, defaultHeaderFormat.SpanID)
+	assert.NotEqual(t, "", req.Header.Get(defaultHeaderFormat.TraceID),
+		"Headers %v should contain %q", req.Header, defaultHeaderFormat.TraceID)
+
 	c, err := tracer.Extract(opentracing.HTTPHeaders, carrier)
 	assert.NoError(t, err)
 
@@ -230,7 +235,6 @@ func TestTracerInjectExtractHeader(t *testing.T) {
 	assert.Equal(t, trace.TraceID, ctx.TraceID())
 
 	assert.Equal(t, trace.SpanID, ctx.SpanID(), "original trace and context should share the same SpanId")
-	assert.Equal(t, trace.Resource, ctx.Resource())
 }
 
 func TestTraceExtractHeaderEnvoy(t *testing.T) {