Merge pull request #1508 from grafana/rollup-indicator-cleanup

Rollup indicator cleanup

Author: Dieterbe

api/ccache_test.go

@@ -24,7 +24,7 @@ func newSrv(delSeries, delArchives int) (*Server, *cache.MockCache) {
 	srv.RegisterRoutes()
 
 	mdata.SetSingleAgg(conf.Avg, conf.Min, conf.Max)
-	mdata.SetSingleSchema(conf.NewRetentionMT(10, 100, 600, 10, 0))
+	mdata.SetSingleSchema(conf.MustParseRetentions("10s:100s:10min:10:true"))
 
 	store := mdata.NewMockStore()
 	store.Drop = true

api/dataprocessor.go

@@ -358,6 +358,7 @@ func (s *Server) getTarget(ctx context.Context, ss *models.StorageStats, req mod
 				// rather than a runtime divide of 2 series
 				SchemaID:              req.SchemaId,
 				Archive:               req.Archive,
+				ArchInterval:          req.ArchInterval,
 				AggNumNorm:            req.AggNum,
 				ConsolidatorNormFetch: req.Consolidator,
 				Count:                 1,

api/dataprocessor_test.go

@@ -364,7 +364,7 @@ func TestGetSeriesFixed(t *testing.T) {
 	store.Drop = true
 
 	mdata.SetSingleAgg(conf.Avg, conf.Min, conf.Max)
-	mdata.SetSingleSchema(conf.NewRetentionMT(10, 100, 600, 10, 0))
+	mdata.SetSingleSchema(conf.MustParseRetentions("10s:100s:10min:10:true"))
 
 	metrics := mdata.NewAggMetrics(store, &cache.MockCache{}, false, nil, 0, 0, 0)
 	srv, _ := NewServer()
@@ -412,7 +412,7 @@ func TestGetSeriesFixedVariableOutInterval(t *testing.T) {
 	store.Drop = true
 
 	mdata.SetSingleAgg(conf.Sum, conf.Avg, conf.Min, conf.Max)
-	mdata.SetSingleSchema(conf.NewRetentionMT(10, 100, 600, 10, 0))
+	mdata.SetSingleSchema(conf.MustParseRetentions("10s:100s:10min:10:true"))
 
 	cache := cache.NewCCache()
 	metrics := mdata.NewAggMetrics(store, cache, false, nil, 0, 0, 0)

api/models/series.go

@@ -9,11 +9,13 @@ import (
 
 	"github.com/grafana/metrictank/consolidation"
 	"github.com/grafana/metrictank/expr/tagquery"
+	"github.com/grafana/metrictank/mdata"
 	"github.com/grafana/metrictank/schema"
 	pickle "github.com/kisielk/og-rek"
 )
 
 //go:generate msgp
+//msgp:ignore SeriesMetaPropertiesExport
 
 type Series struct {
 	Target       string            // for fetched data, set from models.Req.Target, i.e. the metric graphite key. for function output, whatever should be shown as target string (legend)
@@ -37,13 +39,44 @@ type SeriesMeta []SeriesMetaProperties
 type SeriesMetaProperties struct {
 	SchemaID              uint16                     // id of storage-schemas rule this series corresponds to
 	Archive               uint8                      // which archive was being read from
+	ArchInterval          uint32                     // the interval of the archive we fetch
 	AggNumNorm            uint32                     // aggNum for normalization
 	AggNumRC              uint32                     // aggNum runtime consolidation
 	ConsolidatorNormFetch consolidation.Consolidator // consolidator used for normalization and reading from store (if applicable)
 	ConsolidatorRC        consolidation.Consolidator // consolidator used for runtime consolidation to honor maxdatapoints (if applicable).
 	Count                 uint32                     // number of series corresponding to these properties
 }
 
+// SeriesMetaPropertiesExport is an "export" of a SeriesMetaProperties
+// it is a more user friendly representation
+type SeriesMetaPropertiesExport struct {
+	SchemaName            string                     // name of schema rule used
+	SchemaRetentions      string                     // schema retentions used
+	ArchiveRead           uint8                      // which archive was being read from
+	ArchInterval          uint32                     // the interval of the archive we fetch
+	AggNumNorm            uint32                     // aggNum for normalization
+	AggNumRC              uint32                     // aggNum runtime consolidation
+	ConsolidatorNormFetch consolidation.Consolidator // consolidator used for normalization and reading from store (if applicable)
+	ConsolidatorRC        consolidation.Consolidator // consolidator used for runtime consolidation to honor maxdatapoints (if applicable).
+	Count                 uint32                     // number of series corresponding to these properties
+}
+
+// Export returns a human-friendly version of the SeriesMetaProperties.
+func (smp SeriesMetaProperties) Export() SeriesMetaPropertiesExport {
+	schema := mdata.Schemas.Get(smp.SchemaID)
+	return SeriesMetaPropertiesExport{
+		SchemaName:            schema.Name,
+		SchemaRetentions:      schema.Retentions.Orig,
+		ArchiveRead:           smp.Archive,
+		ArchInterval:          smp.ArchInterval,
+		AggNumNorm:            smp.AggNumNorm,
+		AggNumRC:              smp.AggNumRC,
+		ConsolidatorNormFetch: smp.ConsolidatorNormFetch,
+		ConsolidatorRC:        smp.ConsolidatorRC,
+		Count:                 smp.Count,
+	}
+}
+
 // Merge merges SeriesMeta b into a
 // counts for identical properties get added together
 func (a SeriesMeta) Merge(b SeriesMeta) SeriesMeta {
@@ -290,21 +323,25 @@ func (series SeriesByTarget) MarshalJSONFastWithMeta(b []byte) ([]byte, error) {
 func (meta SeriesMeta) MarshalJSONFast(b []byte) ([]byte, error) {
 	b = append(b, '[')
 	for _, props := range meta {
-		b = append(b, `{"schema-id":`...)
-		// TODO make user friendly return the actual rule
-		b = strconv.AppendUint(b, uint64(props.SchemaID), 10)
-		b = append(b, `,"archive":`...)
-		b = strconv.AppendUint(b, uint64(props.Archive), 10)
+		exp := props.Export()
+		b = append(b, `{"schema-name":"`...)
+		b = append(b, exp.SchemaName...)
+		b = append(b, `","schema-retentions":"`...)
+		b = append(b, exp.SchemaRetentions...)
+		b = append(b, `","archive-read":`...)
+		b = strconv.AppendUint(b, uint64(exp.ArchiveRead), 10)
+		b = append(b, `,"archive-interval":`...)
+		b = strconv.AppendUint(b, uint64(exp.ArchInterval), 10)
 		b = append(b, `,"aggnum-norm":`...)
-		b = strconv.AppendUint(b, uint64(props.AggNumNorm), 10)
+		b = strconv.AppendUint(b, uint64(exp.AggNumNorm), 10)
 		b = append(b, `,"consolidate-normfetch":"`...)
-		b = append(b, props.ConsolidatorNormFetch.String()...)
+		b = append(b, exp.ConsolidatorNormFetch.String()...)
 		b = append(b, `","aggnum-rc":`...)
-		b = strconv.AppendUint(b, uint64(props.AggNumRC), 10)
+		b = strconv.AppendUint(b, uint64(exp.AggNumRC), 10)
 		b = append(b, `,"consolidate-rc":"`...)
-		b = append(b, props.ConsolidatorRC.String()...)
+		b = append(b, exp.ConsolidatorRC.String()...)
 		b = append(b, `","count":`...)
-		b = strconv.AppendUint(b, uint64(props.Count), 10)
+		b = strconv.AppendUint(b, uint64(exp.Count), 10)
 		b = append(b, `},`...)
 	}
 	if len(meta) != 0 {

api/models/series_gen.go

@@ -59,7 +59,7 @@ func alignRequests(now, from, to uint32, reqs []models.Req) ([]models.Req, uint3
 	var found bool
 	for i := range reqs {
 		req := &reqs[i]
-		retentions := mdata.Schemas.Get(req.SchemaId).Retentions
+		retentions := mdata.Schemas.Get(req.SchemaId).Retentions.Rets
 		for i, ret := range retentions {
 			// skip non-ready option.
 			if ret.Ready > from {
@@ -116,7 +116,7 @@ func alignRequests(now, from, to uint32, reqs []models.Req) ([]models.Req, uint3
 			// we have to deliver an interval higher than what we originally came up with
 
 			// let's see first if we can deliver it via lower-res rollup archives, if we have any
-			retentions := mdata.Schemas.Get(req.SchemaId).Retentions
+			retentions := mdata.Schemas.Get(req.SchemaId).Retentions.Rets
 			for i, ret := range retentions[req.Archive+1:] {
 				archInterval := uint32(ret.SecondsPerPoint)
 				if interval == archInterval && ret.Ready <= from {