Merge pull request #1423 from grafana/implement_series_lookup_by_meta_tag

Implement series lookup and filtering by meta tag

Author: replay

api/graphite.go

@@ -1333,7 +1333,7 @@ func (s *Server) getMetaTagRecords(ctx *middleware.Context) {
 }
 
 func (s *Server) metaTagRecordUpsert(ctx *middleware.Context, upsertRequest models.MetaTagRecordUpsert) {
-	record, err := tagquery.ParseMetaTagRecord(upsertRequest.MetaTags, upsertRequest.Queries)
+	record, err := tagquery.ParseMetaTagRecord(upsertRequest.MetaTags, upsertRequest.Expressions)
 	if err != nil {
 		response.Write(ctx, response.WrapError(err))
 		return
@@ -1372,7 +1372,7 @@ func (s *Server) metaTagRecordUpsert(ctx *middleware.Context, upsertRequest mode
 	indexUpsertRequest := models.IndexMetaTagRecordUpsert{
 		OrgId:    ctx.OrgId,
 		MetaTags: upsertRequest.MetaTags,
-		Queries:  upsertRequest.Queries,
+		Queries:  upsertRequest.Expressions,
 	}
 
 	results, errors := s.peerQuery(ctx.Req.Context(), indexUpsertRequest, "metaTagRecordUpsert", "/index/metaTags/upsert")

api/models/meta_records.go

@@ -8,15 +8,15 @@ import (
 )
 
 type MetaTagRecordUpsert struct {
-	MetaTags  []string `json:"metaTags" binding:"Required"`
-	Queries   []string `json:"queries" binding:"Required"`
-	Propagate bool     `json:"propagate"`
+	MetaTags    []string `json:"metaTags" binding:"Required"`
+	Expressions []string `json:"expressions" binding:"Required"`
+	Propagate   bool     `json:"propagate"`
 }
 
 func (m MetaTagRecordUpsert) Trace(span opentracing.Span) {
 	span.LogFields(
 		traceLog.String("metaTags", fmt.Sprintf("%q", m.MetaTags)),
-		traceLog.String("queries", fmt.Sprintf("%q", m.Queries)),
+		traceLog.String("expressions", fmt.Sprintf("%q", m.Expressions)),
 		traceLog.Bool("propagate", m.Propagate),
 	)
 }

api/models/meta_records_gen.go

@@ -1,6 +1,7 @@
 package tagquery
 
 import (
+	"encoding/json"
 	"fmt"
 	"regexp"
 	"sort"
@@ -52,6 +53,12 @@ func (e Expressions) Sort() {
 	})
 }
 
+// MarshalJSON satisfies the json.Marshaler interface
+// it is used by the api endpoint /metaTags to list the meta tag records
+func (e Expressions) MarshalJSON() ([]byte, error) {
+	return json.Marshal(e.Strings())
+}
+
 // Expression represents one expression inside a query of one or many expressions.
 // It provides all the necessary methods that are required to do a tag lookup from an index keyed by
 // tags & values, such as the type memory.TagIndex or the type memory.metaTagIndex.
@@ -134,6 +141,14 @@ type Expression interface {
 	// Every valid query must have at least one expression requiring a non-empty value.
 	RequiresNonEmptyValue() bool
 
+	// ResultIsSmallerWhenInverted returns a bool indicating whether the result set after evaluating
+	// this expression will likely be bigger than half of the tested index entries or smaller.
+	// This is never guaranteed to actually be correct, it is only an assumption based on which we
+	// can optimize performance.
+	// F.e. operators = / =~ / __tag=   would return false
+	//      operators != / !=~          would return true
+	ResultIsSmallerWhenInverted() bool
+
 	// Matches takes a string which should either be a tag key or value depending on the return
 	// value of OperatesOnTag(), then it returns whether the given string satisfies this expression
 	Matches(string) bool

expr/tagquery/expression.go

@@ -25,6 +25,11 @@ func (e *expressionCommon) RequiresNonEmptyValue() bool {
 	return true
 }
 
+func (e *expressionCommon) ResultIsSmallerWhenInverted() bool {
+	// by default assume false, unless a concrete type overrides this method
+	return false
+}
+
 func (e *expressionCommon) MatchesExactly() bool {
 	return false
 }

expr/tagquery/expression_common.go

@@ -40,6 +40,10 @@ func (e *expressionMatchAll) RequiresNonEmptyValue() bool {
 	return false
 }
 
+func (e *expressionMatchAll) ResultIsSmallerWhenInverted() bool {
+	return true
+}
+
 func (e *expressionMatchAll) Matches(value string) bool {
 	return true
 }

expr/tagquery/expression_match_all.go

@@ -30,6 +30,10 @@ func (e *expressionNotEqual) RequiresNonEmptyValue() bool {
 	return false
 }
 
+func (e *expressionNotEqual) ResultIsSmallerWhenInverted() bool {
+	return true
+}
+
 func (e *expressionNotEqual) Matches(value string) bool {
 	return value != e.value
 }

expr/tagquery/expression_not_equal.go

@@ -34,6 +34,10 @@ func (e *expressionNotHasTag) RequiresNonEmptyValue() bool {
 	return false
 }
 
+func (e *expressionNotHasTag) ResultIsSmallerWhenInverted() bool {
+	return true
+}
+
 func (e *expressionNotHasTag) Matches(value string) bool {
 	return value != e.key
 }

expr/tagquery/expression_not_has_tag.go

@@ -41,6 +41,10 @@ func (e *expressionNotMatch) RequiresNonEmptyValue() bool {
 	return e.matchesEmpty
 }
 
+func (e *expressionNotMatch) ResultIsSmallerWhenInverted() bool {
+	return true
+}
+
 func (e *expressionNotMatch) Matches(value string) bool {
 	return !e.valueRe.MatchString(value)
 }

expr/tagquery/expression_not_match.go

@@ -3,11 +3,13 @@ package tagquery
 import (
 	"fmt"
 	"strings"
+
+	"github.com/raintank/schema"
 )
 
 type MetaTagRecord struct {
-	MetaTags    Tags
-	Expressions Expressions
+	MetaTags    Tags        `json:"metaTags"`
+	Expressions Expressions `json:"expressions"`
 }
 
 func ParseMetaTagRecord(metaTags []string, expressions []string) (MetaTagRecord, error) {
@@ -24,6 +26,15 @@ func ParseMetaTagRecord(metaTags []string, expressions []string) (MetaTagRecord,
 		return res, err
 	}
 
+	// we don't actually need to instantiate a query at this point, but we want to verify
+	// that it is possible to instantiate a query from the given meta record expressions.
+	// if we can't instantiate a query from the given expressions, then the meta record
+	// upsert request should be considered invalid and should get rejected.
+	_, err = NewQuery(res.Expressions, 0)
+	if err != nil {
+		return res, fmt.Errorf("Failed to instantiate query from given expressions: %s", err)
+	}
+
 	if len(res.Expressions) == 0 {
 		return res, fmt.Errorf("Meta Tag Record must have at least one query")
 	}
@@ -87,3 +98,32 @@ func (m *MetaTagRecord) EqualExpressions(other *MetaTagRecord) bool {
 func (m *MetaTagRecord) HasMetaTags() bool {
 	return len(m.MetaTags) > 0
 }
+
+func (m *MetaTagRecord) GetMetricDefinitionFilter(lookup IdTagLookup) MetricDefinitionFilter {
+	filters := make([]MetricDefinitionFilter, len(m.Expressions))
+	defaultDecisions := make([]FilterDecision, len(m.Expressions))
+	for i, expr := range m.Expressions {
+		filters[i] = expr.GetMetricDefinitionFilter(lookup)
+		defaultDecisions[i] = expr.GetDefaultDecision()
+	}
+
+	return func(id schema.MKey, name string, tags []string) FilterDecision {
+		for i := range filters {
+			decision := filters[i](id, name, tags)
+
+			if decision == None {
+				return defaultDecisions[i]
+			}
+
+			if decision == Pass {
+				continue
+			}
+
+			if decision == Fail {
+				return Fail
+			}
+		}
+
+		return Pass
+	}
+}

expr/tagquery/meta_tag_record.go

@@ -58,3 +58,12 @@ func TestErrorOnParsingMetaTagRecordWithoutQueryy(t *testing.T) {
 		t.Fatalf("Expected an error, but did not get one")
 	}
 }
+
+func TestErrorOnParsingMetaTagRecordWithValidExpressionsButInvalidQuery(t *testing.T) {
+	// the given expression is valid, but as a query this set of expressions is not valid
+	// because every query must have at least one expression requiring a non-empty value
+	_, err := ParseMetaTagRecord([]string{"meta=tag"}, []string{"a!=b"})
+	if err == nil {
+		t.Fatalf("Expected an error, but did not get one")
+	}
+}

expr/tagquery/meta_tag_record_test.go

@@ -1,6 +1,7 @@
 package tagquery
 
 import (
+	"encoding/json"
 	"fmt"
 	"sort"
 	"strings"
@@ -69,6 +70,12 @@ func (t Tags) Strings() []string {
 	return res
 }
 
+// MarshalJSON satisfies the json.Marshaler interface
+// it is used by the api endpoint /metaTags to list the meta tag records
+func (t Tags) MarshalJSON() ([]byte, error) {
+	return json.Marshal(t.Strings())
+}
+
 type Tag struct {
 	Key   string
 	Value string