Support error and status.code searches

CHANGELOG.md

@@ -71,6 +71,7 @@
 * [ENHANCEMENT] Jsonnet: add `$._config.search_enabled`, correctly set `http_api_prefix` in config [#1072](https://github.com/grafana/tempo/pull/1072) (@kvrhdn)
 * [ENHANCEMENT] Performance: Remove WAL contention between ingest and searches [#1076](https://github.com/grafana/tempo/pull/1076) (@mdisibio)
 * [ENHANCEMENT] Include tempo-cli in the release [#1086](https://github.com/grafana/tempo/pull/1086) (@zalegrala)
+* [ENHANCEMENT] Add search on span status [#1093](https://github.com/grafana/tempo/pull/1093) (@mdisibio)
 * [BUGFIX] Update port spec for GCS docker-compose example [#869](https://github.com/grafana/tempo/pull/869) (@zalegrala)
 * [BUGFIX] Fix "magic number" errors and other block mishandling when an ingester forcefully shuts down [#937](https://github.com/grafana/tempo/issues/937) (@mdisibio)
 * [BUGFIX] Fix compactor memory leak [#806](https://github.com/grafana/tempo/pull/806) (@mdisibio)

modules/distributor/search_data.go

@@ -66,6 +66,9 @@ func extractSearchData(trace *tempopb.Trace, id []byte, extractTag extractTagFun
 
 				// Collect for any spans
 				data.AddTag(search.SpanNameTag, s.Name)
+				if s.Status != nil {
+					data.AddTag(search.StatusCodeTag, strconv.FormatInt(int64(s.Status.Code), 10))
+				}
 				data.SetStartTimeUnixNano(s.StartTimeUnixNano)
 				data.SetEndTimeUnixNano(s.EndTimeUnixNano)
 

modules/querier/querier.go

@@ -18,6 +18,7 @@ import (
 	"github.com/grafana/tempo/pkg/model"
 	"github.com/grafana/tempo/pkg/tempopb"
 	"github.com/grafana/tempo/pkg/validation"
+	"github.com/grafana/tempo/tempodb/search"
 	"github.com/opentracing/opentracing-go"
 	ot_log "github.com/opentracing/opentracing-go/log"
 	"github.com/pkg/errors"
@@ -309,6 +310,11 @@ func (q *Querier) SearchTags(ctx context.Context, req *tempopb.SearchTagsRequest
 		}
 	}
 
+	// Extra hard-coded values (for now)
+	for _, k := range search.GetStaticTags() {
+		uniqueMap[k] = struct{}{}
+	}
+
 	// Final response (sorted)
 	resp := &tempopb.SearchTagsResponse{
 		TagNames: make([]string, 0, len(uniqueMap)),
@@ -327,6 +333,14 @@ func (q *Querier) SearchTagValues(ctx context.Context, req *tempopb.SearchTagVal
 		return nil, errors.Wrap(err, "error extracting org id in Querier.SearchTagValues")
 	}
 
+	// Hard-coded values?
+	staticValues := search.GetStaticTagValues(req.TagName)
+	if len(staticValues) > 0 {
+		return &tempopb.SearchTagValuesResponse{
+			TagValues: staticValues,
+		}, nil
+	}
+
 	replicationSet, err := q.ring.GetReplicationSetForOperation(ring.Read)
 	if err != nil {
 		return nil, errors.Wrap(err, "error finding ingesters in Querier.SearchTagValues")

tempodb/search/pipeline.go

@@ -1,11 +1,13 @@
 package search
 
 import (
+	"strconv"
 	"strings"
 	"time"
 
 	"github.com/grafana/tempo/pkg/tempofb"
 	"github.com/grafana/tempo/pkg/tempopb"
+	v1 "github.com/grafana/tempo/pkg/tempopb/trace/v1"
 )
 
 const SecretExhaustiveSearchTag = "x-dbg-exhaustive"
@@ -59,14 +61,8 @@ func NewSearchPipeline(req *tempopb.SearchRequest) Pipeline {
 		vb := make([][]byte, 0, len(req.Tags))
 
 		for k, v := range req.Tags {
-			if k == SecretExhaustiveSearchTag {
-				// Perform an exhaustive search by:
-				// * no block or page filters means all blocks and pages match
-				// * substitute this trace filter instead rejects everything. therefore it never
-				//   quits early due to enough results
-				p.tracefilters = append(p.tracefilters, func(s tempofb.Trace) bool {
-					return false
-				})
+			skip, k, v := p.rewriteTagLookup(k, v)
+			if skip {
 				continue
 			}
 
@@ -91,6 +87,54 @@ func NewSearchPipeline(req *tempopb.SearchRequest) Pipeline {
 	return p
 }
 
+// rewriteTagLookup intercepts certain tag/value lookups and rewrites them. It returns
+// true if the tag lookup should be excluded from the remaining tag/value lookups because
+// the it was rewritten into a different filter altogehter.  Otherwise it returns false,
+// and a new set of tag/value strings to use, which will either be the original inputs
+// or rewritten lookups.
+func (p *Pipeline) rewriteTagLookup(k, v string) (skip bool, newk, newv string) {
+	switch k {
+	case SecretExhaustiveSearchTag:
+		// Perform an exhaustive search by:
+		// * no block or page filters means all blocks and pages match
+		// * substitute this trace filter instead rejects everything. therefore it never
+		//   quits early due to enough results
+		p.tracefilters = append(p.tracefilters, func(s tempofb.Trace) bool {
+			return false
+		})
+		// Skip
+		return true, "", ""
+
+	case ErrorTag:
+		// Convert error=true|false into status.code=2|1
+		b, err := strconv.ParseBool(v)
+		if err != nil {
+			// Not a bool string = fall-through
+			break
+		}
+
+		if b {
+			// Error = true
+			return false, StatusCodeTag, strconv.Itoa(int(v1.Status_STATUS_CODE_ERROR))
+		} else {
+			// Error = false
+			return false, StatusCodeTag, strconv.Itoa(int(v1.Status_STATUS_CODE_OK))
+		}
+
+	case StatusCodeTag:
+		// Convert status.code=string into status.code=int
+		for statusStr, statusID := range StatusCodeValues {
+			if v == statusStr {
+				return false, StatusCodeTag, strconv.Itoa(statusID)
+			}
+		}
+		// Unknown mapping = fall-through
+	}
+
+	// No rewrite
+	return false, k, v
+}
+
 func (p *Pipeline) Matches(e tempofb.Trace) bool {
 
 	for _, f := range p.tracefilters {

tempodb/search/pipeline_test.go

@@ -1,11 +1,13 @@
 package search
 
 import (
+	"strconv"
 	"testing"
 	"time"
 
 	"github.com/grafana/tempo/pkg/tempofb"
 	"github.com/grafana/tempo/pkg/tempopb"
+	v1 "github.com/grafana/tempo/pkg/tempopb/trace/v1"
 	"github.com/stretchr/testify/require"
 )
 
@@ -46,7 +48,20 @@ func TestPipelineMatchesTags(t *testing.T) {
 			searchData:  map[string][]string{"key1": {"value1"}, "key2": {"value2"}},
 			request:     map[string]string{"key1": "value1", "key3": "value3"},
 			shouldMatch: false,
-		}}
+		},
+		{
+			name:        "rewriteError",
+			searchData:  map[string][]string{StatusCodeTag: {strconv.Itoa(int(v1.Status_STATUS_CODE_ERROR))}},
+			request:     map[string]string{"error": "t"},
+			shouldMatch: true,
+		},
+		{
+			name:        "rewriteStatusCode",
+			searchData:  map[string][]string{StatusCodeTag: {strconv.Itoa(int(v1.Status_STATUS_CODE_ERROR))}},
+			request:     map[string]string{StatusCodeTag: StatusCodeError},
+			shouldMatch: true,
+		},
+	}
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {

tempodb/search/util.go

@@ -3,6 +3,7 @@ package search
 import (
 	"github.com/grafana/tempo/pkg/tempofb"
 	"github.com/grafana/tempo/pkg/tempopb"
+	v1 "github.com/grafana/tempo/pkg/tempopb/trace/v1"
 	"github.com/grafana/tempo/pkg/util"
 )
 
@@ -11,8 +12,36 @@ const (
 	ServiceNameTag     = "service.name"
 	RootSpanNameTag    = "root.name"
 	SpanNameTag        = "name"
+	ErrorTag           = "error"
+	StatusCodeTag      = "status.code"
+	StatusCodeUnset    = "unset"
+	StatusCodeOK       = "ok"
+	StatusCodeError    = "error"
 )
 
+var StatusCodeValues = map[string]int{
+	StatusCodeUnset: int(v1.Status_STATUS_CODE_UNSET),
+	StatusCodeOK:    int(v1.Status_STATUS_CODE_OK),
+	StatusCodeError: int(v1.Status_STATUS_CODE_ERROR),
+}
+
+func GetStaticTags() []string {
+	return []string{StatusCodeTag, ErrorTag}
+}
+
+func GetStaticTagValues(tagName string) []string {
+	switch tagName {
+
+	case StatusCodeTag:
+		return []string{StatusCodeUnset, StatusCodeOK, StatusCodeError}
+
+	case ErrorTag:
+		return []string{"true", "false"}
+	}
+
+	return nil
+}
+
 func GetSearchResultFromData(s *tempofb.SearchEntry) *tempopb.TraceSearchMetadata {
 	return &tempopb.TraceSearchMetadata{
 		TraceID:           util.TraceIDToHexString(s.Id()),