From e377292238a01e09d342d849459ad94d0c7bb58f Mon Sep 17 00:00:00 2001
From: Dieter Plaetinck <dieter@grafana.com>
Date: Wed, 11 Apr 2018 14:53:25 +0300
Subject: [PATCH 1/2] schema: make org a uint32
---
vendor/gopkg.in/raintank/schema.v1/metric.go | 4 ++--
vendor/gopkg.in/raintank/schema.v1/metric_gen.go | 10 +++++-----
2 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/vendor/gopkg.in/raintank/schema.v1/metric.go b/vendor/gopkg.in/raintank/schema.v1/metric.go
index 66617063b1..a6668fb868 100644
--- a/vendor/gopkg.in/raintank/schema.v1/metric.go
+++ b/vendor/gopkg.in/raintank/schema.v1/metric.go
@@ -107,7 +107,7 @@ type MetricDataArray []*MetricData
type MetricDefinition struct {
Id MKey `json:"mkey"`
- OrgId int `json:"org_id"`
+ OrgId uint32 `json:"org_id"`
Name string `json:"name"`
Interval int `json:"interval"`
Unit string `json:"unit"`
@@ -235,7 +235,7 @@ func MetricDefinitionFromMetricData(d *MetricData) *MetricDefinition {
md := &MetricDefinition{
Id: mkey,
Name: d.Name,
- OrgId: d.OrgId,
+ OrgId: uint32(d.OrgId),
Mtype: d.Mtype,
Interval: d.Interval,
LastUpdate: d.Time,
diff --git a/vendor/gopkg.in/raintank/schema.v1/metric_gen.go b/vendor/gopkg.in/raintank/schema.v1/metric_gen.go
index 586360d8ab..979ed5e5f0 100644
--- a/vendor/gopkg.in/raintank/schema.v1/metric_gen.go
+++ b/vendor/gopkg.in/raintank/schema.v1/metric_gen.go
@@ -453,7 +453,7 @@ func (z *MetricDefinition) DecodeMsg(dc *msgp.Reader) (err error) {
return
}
case "OrgId":
- z.OrgId, err = dc.ReadInt()
+ z.OrgId, err = dc.ReadUint32()
if err != nil {
return
}
@@ -531,7 +531,7 @@ func (z *MetricDefinition) EncodeMsg(en *msgp.Writer) (err error) {
if err != nil {
return
}
- err = en.WriteInt(z.OrgId)
+ err = en.WriteUint32(z.OrgId)
if err != nil {
return
}
@@ -619,7 +619,7 @@ func (z *MetricDefinition) MarshalMsg(b []byte) (o []byte, err error) {
}
// string "OrgId"
o = append(o, 0xa5, 0x4f, 0x72, 0x67, 0x49, 0x64)
- o = msgp.AppendInt(o, z.OrgId)
+ o = msgp.AppendUint32(o, z.OrgId)
// string "Name"
o = append(o, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
o = msgp.AppendString(o, z.Name)
@@ -669,7 +669,7 @@ func (z *MetricDefinition) UnmarshalMsg(bts []byte) (o []byte, err error) {
return
}
case "OrgId":
- z.OrgId, bts, err = msgp.ReadIntBytes(bts)
+ z.OrgId, bts, err = msgp.ReadUint32Bytes(bts)
if err != nil {
return
}
@@ -733,7 +733,7 @@ func (z *MetricDefinition) UnmarshalMsg(bts []byte) (o []byte, err error) {
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z *MetricDefinition) Msgsize() (s int) {
- s = 1 + 3 + z.Id.Msgsize() + 6 + msgp.IntSize + 5 + msgp.StringPrefixSize + len(z.Name) + 9 + msgp.IntSize + 5 + msgp.StringPrefixSize + len(z.Unit) + 6 + msgp.StringPrefixSize + len(z.Mtype) + 5 + msgp.ArrayHeaderSize
+ s = 1 + 3 + z.Id.Msgsize() + 6 + msgp.Uint32Size + 5 + msgp.StringPrefixSize + len(z.Name) + 9 + msgp.IntSize + 5 + msgp.StringPrefixSize + len(z.Unit) + 6 + msgp.StringPrefixSize + len(z.Mtype) + 5 + msgp.ArrayHeaderSize
for za0001 := range z.Tags {
s += msgp.StringPrefixSize + len(z.Tags[za0001])
}
From 076fa4cfced9fddce001bf43ecc2574bd94f1c49 Mon Sep 17 00:00:00 2001
From: Dieter Plaetinck <dieter@grafana.com>
Date: Wed, 11 Apr 2018 15:19:21 +0300
Subject: [PATCH 2/2] represent orgId as uint32 instead of int
should result in memory savings in the index
when loading for cassandra, if <0 -> assign OrgIdPublic
---
api/graphite.go | 26 +++++++++----------
api/middleware/middleware.go | 8 +++---
api/models/ccache.go | 2 +-
api/models/node.go | 18 ++++++-------
api/prometheus_querier.go | 6 ++---
cmd/mt-index-migrate/main.go | 2 +-
idx/cassandra/cassandra.go | 7 +++--
idx/cassandra/cassandra_test.go | 6 ++---
idx/idx.go | 22 ++++++++--------
idx/memory/memory.go | 46 ++++++++++++++++-----------------
idx/memory/memory_find_test.go | 20 +++++++-------
idx/memory/memory_test.go | 6 ++---
metrictank.go | 2 +-
13 files changed, 87 insertions(+), 84 deletions(-)
diff --git a/api/graphite.go b/api/graphite.go
index b0c5581d8e..90fe3e6e7f 100644
--- a/api/graphite.go
+++ b/api/graphite.go
@@ -54,7 +54,7 @@ type Series struct {
Node cluster.Node
}
-func (s *Server) findSeries(ctx context.Context, orgId int, patterns []string, seenAfter int64) ([]Series, error) {
+func (s *Server) findSeries(ctx context.Context, orgId uint32, patterns []string, seenAfter int64) ([]Series, error) {
peers, err := cluster.MembersForQuery()
if err != nil {
log.Error(3, "HTTP findSeries unable to get peers, %s", err)
@@ -118,7 +118,7 @@ func (s *Server) findSeries(ctx context.Context, orgId int, patterns []string, s
return series, nil
}
-func (s *Server) findSeriesLocal(ctx context.Context, orgId int, patterns []string, seenAfter int64) ([]Series, error) {
+func (s *Server) findSeriesLocal(ctx context.Context, orgId uint32, patterns []string, seenAfter int64) ([]Series, error) {
result := make([]Series, 0)
for _, pattern := range patterns {
select {
@@ -149,7 +149,7 @@ func (s *Server) findSeriesLocal(ctx context.Context, orgId int, patterns []stri
return result, nil
}
-func (s *Server) findSeriesRemote(ctx context.Context, orgId int, patterns []string, seenAfter int64, peer cluster.Node) ([]Series, error) {
+func (s *Server) findSeriesRemote(ctx context.Context, orgId uint32, patterns []string, seenAfter int64, peer cluster.Node) ([]Series, error) {
log.Debug("HTTP Render querying %s/index/find for %d:%q", peer.GetName(), orgId, patterns)
data := models.IndexFind{
Patterns: patterns,
@@ -370,11 +370,11 @@ func (s *Server) metricsFind(ctx *middleware.Context, request models.GraphiteFin
}
}
-func (s *Server) listLocal(orgId int) []idx.Archive {
+func (s *Server) listLocal(orgId uint32) []idx.Archive {
return s.MetricIndex.List(orgId)
}
-func (s *Server) listRemote(ctx context.Context, orgId int, peer cluster.Node) ([]idx.Archive, error) {
+func (s *Server) listRemote(ctx context.Context, orgId uint32, peer cluster.Node) ([]idx.Archive, error) {
log.Debug("HTTP IndexJson() querying %s/index/list for %d", peer.GetName(), orgId)
buf, err := peer.Post(ctx, "listRemote", "/index/list", models.IndexList{OrgId: orgId})
if err != nil {
@@ -635,12 +635,12 @@ func (s *Server) metricsDelete(ctx *middleware.Context, req models.MetricsDelete
response.Write(ctx, response.NewJson(200, resp, ""))
}
-func (s *Server) metricsDeleteLocal(orgId int, query string) (int, error) {
+func (s *Server) metricsDeleteLocal(orgId uint32, query string) (int, error) {
defs, err := s.MetricIndex.Delete(orgId, query)
return len(defs), err
}
-func (s *Server) metricsDeleteRemote(ctx context.Context, orgId int, query string, peer cluster.Node) (int, error) {
+func (s *Server) metricsDeleteRemote(ctx context.Context, orgId uint32, query string, peer cluster.Node) (int, error) {
log.Debug("HTTP metricDelete calling %s/index/delete for %d:%q", peer.GetName(), orgId, query)
body := models.IndexDelete{
@@ -673,7 +673,7 @@ func (s *Server) metricsDeleteRemote(ctx context.Context, orgId int, query strin
// executePlan looks up the needed data, retrieves it, and then invokes the processing
// note if you do something like sum(foo.*) and all of those metrics happen to be on another node,
// we will collect all the indidividual series from the peer, and then sum here. that could be optimized
-func (s *Server) executePlan(ctx context.Context, orgId int, plan expr.Plan) ([]models.Series, error) {
+func (s *Server) executePlan(ctx context.Context, orgId uint32, plan expr.Plan) ([]models.Series, error) {
minFrom := uint32(math.MaxUint32)
var maxTo uint32
@@ -861,7 +861,7 @@ func (s *Server) graphiteTagDetails(ctx *middleware.Context, request models.Grap
response.Write(ctx, response.NewJson(200, resp, ""))
}
-func (s *Server) clusterTagDetails(ctx context.Context, orgId int, tag, filter string, from int64) (map[string]uint64, error) {
+func (s *Server) clusterTagDetails(ctx context.Context, orgId uint32, tag, filter string, from int64) (map[string]uint64, error) {
result, err := s.MetricIndex.TagDetails(orgId, tag, filter, from)
if err != nil {
return nil, err
@@ -923,7 +923,7 @@ func (s *Server) graphiteTagFindSeries(ctx *middleware.Context, request models.G
response.Write(ctx, response.NewJson(200, seriesNames, ""))
}
-func (s *Server) clusterFindByTag(ctx context.Context, orgId int, expressions []string, from int64) ([]Series, error) {
+func (s *Server) clusterFindByTag(ctx context.Context, orgId uint32, expressions []string, from int64) ([]Series, error) {
seriesSet := make(map[string]Series)
result, err := s.MetricIndex.FindByTag(orgId, expressions, from)
@@ -1005,7 +1005,7 @@ func (s *Server) graphiteTags(ctx *middleware.Context, request models.GraphiteTa
response.Write(ctx, response.NewJson(200, resp, ""))
}
-func (s *Server) clusterTags(ctx context.Context, orgId int, filter string, from int64) ([]string, error) {
+func (s *Server) clusterTags(ctx context.Context, orgId uint32, filter string, from int64) ([]string, error) {
result, err := s.MetricIndex.Tags(orgId, filter, from)
if err != nil {
return nil, err
@@ -1068,7 +1068,7 @@ func (s *Server) graphiteAutoCompleteTags(ctx *middleware.Context, request model
response.Write(ctx, response.NewJson(200, tags, ""))
}
-func (s *Server) clusterAutoCompleteTags(ctx context.Context, orgId int, prefix string, expressions []string, from int64, limit uint) ([]string, error) {
+func (s *Server) clusterAutoCompleteTags(ctx context.Context, orgId uint32, prefix string, expressions []string, from int64, limit uint) ([]string, error) {
result, err := s.MetricIndex.FindTags(orgId, prefix, expressions, from, limit)
if err != nil {
return nil, err
@@ -1123,7 +1123,7 @@ func (s *Server) graphiteAutoCompleteTagValues(ctx *middleware.Context, request
response.Write(ctx, response.NewJson(200, resp, ""))
}
-func (s *Server) clusterAutoCompleteTagValues(ctx context.Context, orgId int, tag, prefix string, expressions []string, from int64, limit uint) ([]string, error) {
+func (s *Server) clusterAutoCompleteTagValues(ctx context.Context, orgId uint32, tag, prefix string, expressions []string, from int64, limit uint) ([]string, error) {
result, err := s.MetricIndex.FindTagValues(orgId, tag, prefix, expressions, from, limit)
if err != nil {
return nil, err
diff --git a/api/middleware/middleware.go b/api/middleware/middleware.go
index 4f1bc1e467..d7d4143829 100644
--- a/api/middleware/middleware.go
+++ b/api/middleware/middleware.go
@@ -12,7 +12,7 @@ import (
type Context struct {
*macaron.Context
- OrgId int
+ OrgId uint32
Body io.ReadCloser
}
@@ -31,7 +31,7 @@ func OrgMiddleware(multiTenant bool) macaron.Handler {
}
}
-func getOrg(req *http.Request, multiTenant bool) (int, error) {
+func getOrg(req *http.Request, multiTenant bool) (uint32, error) {
if !multiTenant {
return 1, nil
}
@@ -40,10 +40,10 @@ func getOrg(req *http.Request, multiTenant bool) (int, error) {
return 0, nil
}
org, err := strconv.Atoi(orgStr)
- if err != nil {
+ if err != nil || org < 1 {
return 0, errors.New("bad org-id")
}
- return org, nil
+ return uint32(org), nil
}
func RequireOrg() macaron.Handler {
diff --git a/api/models/ccache.go b/api/models/ccache.go
index dc397bbb13..986898f94d 100644
--- a/api/models/ccache.go
+++ b/api/models/ccache.go
@@ -9,7 +9,7 @@ type CCacheDelete struct {
Patterns []string `json:"patterns" form:"patterns" `
// tag expressions to select series
Expr []string `json:"expr" form:"expr"`
- OrgId int `json:"orgId" form:"orgId" binding:"Required"`
+ OrgId uint32 `json:"orgId" form:"orgId" binding:"Required"`
Propagate bool `json:"propagate" form:"propagate"`
}
diff --git a/api/models/node.go b/api/models/node.go
index d7674c81c7..a71600be93 100644
--- a/api/models/node.go
+++ b/api/models/node.go
@@ -26,7 +26,7 @@ type ClusterMembersResp struct {
}
type IndexList struct {
- OrgId int `json:"orgId" form:"orgId" binding:"Required"`
+ OrgId uint32 `json:"orgId" form:"orgId" binding:"Required"`
}
func (i IndexList) Trace(span opentracing.Span) {
@@ -37,7 +37,7 @@ func (i IndexList) TraceDebug(span opentracing.Span) {
}
type IndexFindByTag struct {
- OrgId int `json:"orgId" binding:"Required"`
+ OrgId uint32 `json:"orgId" binding:"Required"`
Expr []string `json:"expressions"`
From int64 `json:"from"`
}
@@ -52,7 +52,7 @@ func (i IndexFindByTag) TraceDebug(span opentracing.Span) {
}
type IndexTagDetails struct {
- OrgId int `json:"orgId" binding:"Required"`
+ OrgId uint32 `json:"orgId" binding:"Required"`
Filter string `json:"filter"`
Tag string `json:"tag" binding:"Required"`
From int64 `json:"from"`
@@ -69,7 +69,7 @@ func (i IndexTagDetails) TraceDebug(span opentracing.Span) {
}
type IndexTags struct {
- OrgId int `json:"orgId" binding:"Required"`
+ OrgId uint32 `json:"orgId" binding:"Required"`
Filter string `json:"filter"`
From int64 `json:"from"`
}
@@ -84,7 +84,7 @@ func (i IndexTags) TraceDebug(span opentracing.Span) {
}
type IndexAutoCompleteTags struct {
- OrgId int `json:"orgId" binding:"Required"`
+ OrgId uint32 `json:"orgId" binding:"Required"`
Prefix string `json:"Prefix"`
Expr []string `json:"expressions"`
From int64 `json:"from"`
@@ -103,7 +103,7 @@ func (i IndexAutoCompleteTags) TraceDebug(span opentracing.Span) {
}
type IndexAutoCompleteTagValues struct {
- OrgId int `json:"orgId" binding:"Required"`
+ OrgId uint32 `json:"orgId" binding:"Required"`
Tag string `json:"tag"`
Prefix string `json:"prefix"`
Expr []string `json:"expressions"`
@@ -124,7 +124,7 @@ func (i IndexAutoCompleteTagValues) TraceDebug(span opentracing.Span) {
}
type IndexTagDelSeries struct {
- OrgId int `json:"orgId" binding:"Required"`
+ OrgId uint32 `json:"orgId" binding:"Required"`
Paths []string `json:"path" form:"path"`
}
@@ -142,7 +142,7 @@ type IndexGet struct {
type IndexFind struct {
Patterns []string `json:"patterns" form:"patterns" binding:"Required"`
- OrgId int `json:"orgId" form:"orgId" binding:"Required"`
+ OrgId uint32 `json:"orgId" form:"orgId" binding:"Required"`
From int64 `json:"from" form:"from"`
}
@@ -177,7 +177,7 @@ func (g GetData) TraceDebug(span opentracing.Span) {
type IndexDelete struct {
Query string `json:"query" form:"query" binding:"Required"`
- OrgId int `json:"orgId" form:"orgId" binding:"Required"`
+ OrgId uint32 `json:"orgId" form:"orgId" binding:"Required"`
}
func (i IndexDelete) Trace(span opentracing.Span) {
diff --git a/api/prometheus_querier.go b/api/prometheus_querier.go
index ba6579c110..4f14b93472 100644
--- a/api/prometheus_querier.go
+++ b/api/prometheus_querier.go
@@ -21,7 +21,7 @@ import (
func (s *Server) Querier(ctx context.Context, min, max int64) (storage.Querier, error) {
from := uint32(min / 1000)
to := uint32(max / 1000)
- return NewQuerier(ctx, s, from, to, ctx.Value(orgID("org-id")).(int), false), nil
+ return NewQuerier(ctx, s, from, to, ctx.Value(orgID("org-id")).(uint32), false), nil
}
// querier implements Prometheus' Querier interface
@@ -29,12 +29,12 @@ type querier struct {
*Server
from uint32
to uint32
- OrgID int
+ OrgID uint32
metadataOnly bool
ctx context.Context
}
-func NewQuerier(ctx context.Context, s *Server, from, to uint32, orgId int, metadataOnly bool) storage.Querier {
+func NewQuerier(ctx context.Context, s *Server, from, to uint32, orgId uint32, metadataOnly bool) storage.Querier {
return &querier{
s,
from,
diff --git a/cmd/mt-index-migrate/main.go b/cmd/mt-index-migrate/main.go
index 937f771c1f..37dd896e3e 100644
--- a/cmd/mt-index-migrate/main.go
+++ b/cmd/mt-index-migrate/main.go
@@ -156,7 +156,7 @@ func getDefs(session *gocql.Session, defsChan chan *schema.MetricDefinition) {
}
mdef := schema.MetricDefinition{
Id: mkey,
- OrgId: orgId,
+ OrgId: uint32(orgId),
Partition: partition,
Name: name,
Interval: interval,
diff --git a/idx/cassandra/cassandra.go b/idx/cassandra/cassandra.go
index e14913e4b2..12370ca5ba 100644
--- a/idx/cassandra/cassandra.go
+++ b/idx/cassandra/cassandra.go
@@ -399,10 +399,13 @@ func (c *CasIdx) load(defs []schema.MetricDefinition, iter cqlIterator, cutoff u
log.Error(3, "cassandra-idx: load() could not parse ID %q: %s -> skipping", id, err)
continue
}
+ if orgId < 0 {
+ orgId = int(idx.OrgIdPublic)
+ }
mdef := &schema.MetricDefinition{
Id: mkey,
- OrgId: orgId,
+ OrgId: uint32(orgId),
Partition: partition,
Name: name,
Interval: interval,
@@ -487,7 +490,7 @@ func (c *CasIdx) processWriteQueue() {
c.wg.Done()
}
-func (c *CasIdx) Delete(orgId int, pattern string) ([]idx.Archive, error) {
+func (c *CasIdx) Delete(orgId uint32, pattern string) ([]idx.Archive, error) {
pre := time.Now()
defs, err := c.MemoryIdx.Delete(orgId, pattern)
if err != nil {
diff --git a/idx/cassandra/cassandra_test.go b/idx/cassandra/cassandra_test.go
index f4c1679a68..4e37ccfa80 100644
--- a/idx/cassandra/cassandra_test.go
+++ b/idx/cassandra/cassandra_test.go
@@ -104,14 +104,14 @@ func getRandomString(n int, alphabets ...byte) string {
return string(bytes)
}
-func getMetricData(orgId, depth, count, interval int, prefix string) []*schema.MetricData {
+func getMetricData(orgId uint32, depth, count, interval int, prefix string) []*schema.MetricData {
data := make([]*schema.MetricData, count)
series := getSeriesNames(depth, count, prefix)
for i, s := range series {
data[i] = &schema.MetricData{
Name: s,
- OrgId: orgId,
+ OrgId: int(orgId),
Interval: interval,
}
data[i].SetId()
@@ -131,7 +131,7 @@ func TestGetAddKey(t *testing.T) {
org2Series := getMetricData(2, 2, 5, 10, "metric.org2")
for _, series := range [][]*schema.MetricData{publicSeries, org1Series, org2Series} {
- orgId := series[0].OrgId
+ orgId := uint32(series[0].OrgId)
Convey(fmt.Sprintf("When indexing metrics for orgId %d", orgId), t, func() {
for _, s := range series {
mkey, err := schema.MKeyFromString(s.Id)
diff --git a/idx/idx.go b/idx/idx.go
index 4917ba7968..798df783c7 100644
--- a/idx/idx.go
+++ b/idx/idx.go
@@ -9,7 +9,7 @@ import (
schema "gopkg.in/raintank/schema.v1"
)
-var OrgIdPublic = 0
+var OrgIdPublic = uint32(0)
var (
BothBranchAndLeaf = errors.New("node can't be both branch and leaf")
@@ -69,23 +69,23 @@ type MetricIndex interface {
Get(key schema.MKey) (Archive, bool)
// GetPath returns the archives under the given path.
- GetPath(orgId int, path string) []Archive
+ GetPath(orgId uint32, path string) []Archive
// Delete deletes items from the index
// If the pattern matches a branch node, then
// all leaf nodes on that branch are deleted. So if the pattern is
// "*", all items in the index are deleted.
// It returns a copy of all of the Archives deleted.
- Delete(orgId int, pattern string) ([]Archive, error)
+ Delete(orgId uint32, pattern string) ([]Archive, error)
// Find searches the index for matching nodes.
// * orgId describes the org to search in (public data in orgIdPublic is automatically included)
// * pattern is handled like graphite does. see https://graphite.readthedocs.io/en/latest/render_api.html#paths-and-wildcards
// * from is a unix timestamp. series not updated since then are excluded.
- Find(orgId int, pattern string, from int64) ([]Node, error)
+ Find(orgId uint32, pattern string, from int64) ([]Node, error)
// List returns all Archives for the passed OrgId and the public orgId
- List(orgId int) []Archive
+ List(orgId uint32) []Archive
// Prune deletes all metrics that haven't been seen since the given timestamp.
// It returns all Archives deleted and any error encountered.
@@ -99,24 +99,24 @@ type MetricIndex interface {
// where the LastUpdate time is >= from will be returned as results.
// The returned results are not deduplicated and in certain cases it is possible
// that duplicate entries will be returned.
- FindByTag(orgId int, expressions []string, from int64) ([]Node, error)
+ FindByTag(orgId uint32, expressions []string, from int64) ([]Node, error)
// Tags returns a list of all tag keys associated with the metrics of a given
// organization. The return values are filtered by the regex in the second parameter.
// If the third parameter is >0 then only metrics will be accounted of which the
// LastUpdate time is >= the given value.
- Tags(orgId int, filter string, from int64) ([]string, error)
+ Tags(orgId uint32, filter string, from int64) ([]string, error)
// FindTags generates a list of possible tags that could complete a
// given prefix. It also accepts additional tag conditions to further narrow
// down the result set in the format of graphite's tag queries
- FindTags(orgId int, prefix string, expressions []string, from int64, limit uint) ([]string, error)
+ FindTags(orgId uint32, prefix string, expressions []string, from int64, limit uint) ([]string, error)
// FindTagValues generates a list of possible values that could
// complete a given value prefix. It requires a tag to be specified and only values
// of the given tag will be returned. It also accepts additional conditions to
// further narrow down the result set in the format of graphite's tag queries
- FindTagValues(orgId int, tag string, prefix string, expressions []string, from int64, limit uint) ([]string, error)
+ FindTagValues(orgId uint32, tag string, prefix string, expressions []string, from int64, limit uint) ([]string, error)
// TagDetails returns a list of all values associated with a given tag key in the
// given org. The occurrences of each value is counted and the count is referred to by
@@ -125,9 +125,9 @@ type MetricIndex interface {
// the values before accounting for them.
// If the fourth parameter is > 0 then only those metrics of which the LastUpdate
// time is >= the from timestamp will be included.
- TagDetails(orgId int, key string, filter string, from int64) (map[string]uint64, error)
+ TagDetails(orgId uint32, key string, filter string, from int64) (map[string]uint64, error)
// DeleteTagged deletes the specified series from the tag index and also the
// DefById index.
- DeleteTagged(orgId int, paths []string) ([]Archive, error)
+ DeleteTagged(orgId uint32, paths []string) ([]Archive, error)
}
diff --git a/idx/memory/memory.go b/idx/memory/memory.go
index 629b53a6c3..cc24091c49 100755
--- a/idx/memory/memory.go
+++ b/idx/memory/memory.go
@@ -108,7 +108,7 @@ func (t *TagIndex) delTagId(name, value string, id schema.MKey) {
// org id -> nameWithTags -> Set of references to schema.MetricDefinition
// nameWithTags is the name plus all tags in the <name>;<tag>=<value>... format.
-type defByTagSet map[int]map[string]map[*schema.MetricDefinition]struct{}
+type defByTagSet map[uint32]map[string]map[*schema.MetricDefinition]struct{}
func (defs defByTagSet) add(def *schema.MetricDefinition) {
var orgDefs map[string]map[*schema.MetricDefinition]struct{}
@@ -144,7 +144,7 @@ func (defs defByTagSet) del(def *schema.MetricDefinition) {
}
}
-func (defs defByTagSet) defs(id int, fullName string) map[*schema.MetricDefinition]struct{} {
+func (defs defByTagSet) defs(id uint32, fullName string) map[*schema.MetricDefinition]struct{} {
var orgDefs map[string]map[*schema.MetricDefinition]struct{}
var ok bool
if orgDefs, ok = defs[id]; !ok {
@@ -184,19 +184,19 @@ type MemoryIdx struct {
defById map[schema.MKey]*idx.Archive
// used by hierarchy index only
- tree map[int]*Tree // by orgId
+ tree map[uint32]*Tree // by orgId
// used by tag index
defByTagSet defByTagSet
- tags map[int]TagIndex // by orgId
+ tags map[uint32]TagIndex // by orgId
}
func New() *MemoryIdx {
return &MemoryIdx{
defById: make(map[schema.MKey]*idx.Archive),
defByTagSet: make(defByTagSet),
- tree: make(map[int]*Tree),
- tags: make(map[int]TagIndex),
+ tree: make(map[uint32]*Tree),
+ tags: make(map[uint32]TagIndex),
}
}
@@ -470,7 +470,7 @@ func (m *MemoryIdx) Get(id schema.MKey) (idx.Archive, bool) {
// GetPath returns the node under the given org and path.
// this is an alternative to Find for when you have a path, not a pattern, and want to lookup in a specific org tree only.
-func (m *MemoryIdx) GetPath(orgId int, path string) []idx.Archive {
+func (m *MemoryIdx) GetPath(orgId uint32, path string) []idx.Archive {
m.RLock()
defer m.RUnlock()
tree, ok := m.tree[orgId]
@@ -489,7 +489,7 @@ func (m *MemoryIdx) GetPath(orgId int, path string) []idx.Archive {
return archives
}
-func (m *MemoryIdx) TagDetails(orgId int, key, filter string, from int64) (map[string]uint64, error) {
+func (m *MemoryIdx) TagDetails(orgId uint32, key, filter string, from int64) (map[string]uint64, error) {
if !TagSupport {
log.Warn("memory-idx: received tag query, but tag support is disabled")
return nil, nil
@@ -561,7 +561,7 @@ func (m *MemoryIdx) TagDetails(orgId int, key, filter string, from int64) (map[s
// limit: the maximum number of results to return
//
// the results will always be sorted alphabetically for consistency
-func (m *MemoryIdx) FindTags(orgId int, prefix string, expressions []string, from int64, limit uint) ([]string, error) {
+func (m *MemoryIdx) FindTags(orgId uint32, prefix string, expressions []string, from int64, limit uint) ([]string, error) {
if !TagSupport {
log.Warn("memory-idx: received tag query, but tag support is disabled")
return nil, nil
@@ -644,7 +644,7 @@ func (m *MemoryIdx) FindTags(orgId int, prefix string, expressions []string, fro
// limit: the maximum number of results to return
//
// the results will always be sorted alphabetically for consistency
-func (m *MemoryIdx) FindTagValues(orgId int, tag, prefix string, expressions []string, from int64, limit uint) ([]string, error) {
+func (m *MemoryIdx) FindTagValues(orgId uint32, tag, prefix string, expressions []string, from int64, limit uint) ([]string, error) {
if !TagSupport {
log.Warn("memory-idx: received tag query, but tag support is disabled")
return nil, nil
@@ -747,7 +747,7 @@ func (m *MemoryIdx) FindTagValues(orgId int, tag, prefix string, expressions []s
// organization. The return values are filtered by the regex in the second parameter.
// If the third parameter is >0 then only metrics will be accounted of which the
// LastUpdate time is >= the given value.
-func (m *MemoryIdx) Tags(orgId int, filter string, from int64) ([]string, error) {
+func (m *MemoryIdx) Tags(orgId uint32, filter string, from int64) ([]string, error) {
if !TagSupport {
log.Warn("memory-idx: received tag query, but tag support is disabled")
return nil, nil
@@ -817,7 +817,7 @@ func (m *MemoryIdx) hasOneMetricFrom(tags TagIndex, tag string, from int64) bool
return false
}
-func (m *MemoryIdx) FindByTag(orgId int, expressions []string, from int64) ([]idx.Node, error) {
+func (m *MemoryIdx) FindByTag(orgId uint32, expressions []string, from int64) ([]idx.Node, error) {
if !TagSupport {
log.Warn("memory-idx: received tag query, but tag support is disabled")
return nil, nil
@@ -851,7 +851,7 @@ func (m *MemoryIdx) FindByTag(orgId int, expressions []string, from int64) ([]id
return res, nil
}
-func (m *MemoryIdx) idsByTagQuery(orgId int, query TagQuery) IdSet {
+func (m *MemoryIdx) idsByTagQuery(orgId uint32, query TagQuery) IdSet {
tags, ok := m.tags[orgId]
if !ok {
return nil
@@ -860,7 +860,7 @@ func (m *MemoryIdx) idsByTagQuery(orgId int, query TagQuery) IdSet {
return query.Run(tags, m.defById)
}
-func (m *MemoryIdx) Find(orgId int, pattern string, from int64) ([]idx.Node, error) {
+func (m *MemoryIdx) Find(orgId uint32, pattern string, from int64) ([]idx.Node, error) {
pre := time.Now()
m.RLock()
defer m.RUnlock()
@@ -914,7 +914,7 @@ func (m *MemoryIdx) Find(orgId int, pattern string, from int64) ([]idx.Node, err
return results, nil
}
-func (m *MemoryIdx) find(orgId int, pattern string) ([]*Node, error) {
+func (m *MemoryIdx) find(orgId uint32, pattern string) ([]*Node, error) {
tree, ok := m.tree[orgId]
if !ok {
log.Debug("memory-idx: orgId %d has no metrics indexed.", orgId)
@@ -1006,7 +1006,7 @@ func (m *MemoryIdx) find(orgId int, pattern string) ([]*Node, error) {
return children, nil
}
-func (m *MemoryIdx) List(orgId int) []idx.Archive {
+func (m *MemoryIdx) List(orgId uint32) []idx.Archive {
pre := time.Now()
m.RLock()
defer m.RUnlock()
@@ -1023,7 +1023,7 @@ func (m *MemoryIdx) List(orgId int) []idx.Archive {
return defs
}
-func (m *MemoryIdx) DeleteTagged(orgId int, paths []string) ([]idx.Archive, error) {
+func (m *MemoryIdx) DeleteTagged(orgId uint32, paths []string) ([]idx.Archive, error) {
if !TagSupport {
log.Warn("memory-idx: received tag query, but tag support is disabled")
return nil, nil
@@ -1064,7 +1064,7 @@ func (m *MemoryIdx) DeleteTagged(orgId int, paths []string) ([]idx.Archive, erro
// deleteTaggedByIdSet deletes a map of ids from the tag index and also the DefByIds
// it is important that only IDs of series with tags get passed in here, because
// otherwise the result might be inconsistencies between DefByIDs and the tree index.
-func (m *MemoryIdx) deleteTaggedByIdSet(orgId int, ids IdSet) []idx.Archive {
+func (m *MemoryIdx) deleteTaggedByIdSet(orgId uint32, ids IdSet) []idx.Archive {
tags, ok := m.tags[orgId]
if !ok {
return nil
@@ -1091,7 +1091,7 @@ func (m *MemoryIdx) deleteTaggedByIdSet(orgId int, ids IdSet) []idx.Archive {
return deletedDefs
}
-func (m *MemoryIdx) Delete(orgId int, pattern string) ([]idx.Archive, error) {
+func (m *MemoryIdx) Delete(orgId uint32, pattern string) ([]idx.Archive, error) {
var deletedDefs []idx.Archive
pre := time.Now()
m.Lock()
@@ -1111,7 +1111,7 @@ func (m *MemoryIdx) Delete(orgId int, pattern string) ([]idx.Archive, error) {
return deletedDefs, nil
}
-func (m *MemoryIdx) delete(orgId int, n *Node, deleteEmptyParents, deleteChildren bool) []idx.Archive {
+func (m *MemoryIdx) delete(orgId uint32, n *Node, deleteEmptyParents, deleteChildren bool) []idx.Archive {
tree := m.tree[orgId]
deletedDefs := make([]idx.Archive, 0)
if deleteChildren && n.HasChildren() {
@@ -1208,7 +1208,7 @@ func (m *MemoryIdx) delete(orgId int, n *Node, deleteEmptyParents, deleteChildre
// delete series from the index if they have not been seen since "oldest"
func (m *MemoryIdx) Prune(oldest time.Time) ([]idx.Archive, error) {
oldestUnix := oldest.Unix()
- orgs := make(map[int]struct{})
+ orgs := make(map[uint32]struct{})
log.Info("memory-idx: pruning stale metricDefs across all orgs")
m.RLock()
for org := range m.tree {
@@ -1222,8 +1222,8 @@ func (m *MemoryIdx) Prune(oldest time.Time) ([]idx.Archive, error) {
m.RUnlock()
var pruned []idx.Archive
- toPruneUntagged := make(map[int]map[string]struct{}, len(orgs))
- toPruneTagged := make(map[int]IdSet, len(orgs))
+ toPruneUntagged := make(map[uint32]map[string]struct{}, len(orgs))
+ toPruneTagged := make(map[uint32]IdSet, len(orgs))
for org := range orgs {
toPruneTagged[org] = make(IdSet)
toPruneUntagged[org] = make(map[string]struct{})
diff --git a/idx/memory/memory_find_test.go b/idx/memory/memory_find_test.go
index 34ff4ab1c0..e59a1dc849 100644
--- a/idx/memory/memory_find_test.go
+++ b/idx/memory/memory_find_test.go
@@ -686,7 +686,7 @@ func BenchmarkTagsWithoutFromNorFilter(b *testing.B) {
}
}
-func ixFind(b *testing.B, org, q int) {
+func ixFind(b *testing.B, org uint32, q int) {
b.Helper()
nodes, err := ix.Find(org, queries[q].Pattern, 0)
if err != nil {
@@ -707,14 +707,14 @@ func BenchmarkFind(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
q := n % queryCount
- org := (n % 2) + 1
+ org := uint32((n % 2) + 1)
ixFind(b, org, q)
}
}
type testQ struct {
q int
- org int
+ org uint32
}
func BenchmarkConcurrent4Find(b *testing.B) {
@@ -733,7 +733,7 @@ func BenchmarkConcurrent4Find(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
q := n % queryCount
- org := (n % 2) + 1
+ org := uint32((n % 2) + 1)
ch <- testQ{q: q, org: org}
}
close(ch)
@@ -755,13 +755,13 @@ func BenchmarkConcurrent8Find(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
q := n % queryCount
- org := (n % 2) + 1
+ org := uint32((n % 2) + 1)
ch <- testQ{q: q, org: org}
}
close(ch)
}
-func ixFindByTag(b *testing.B, org, q int) {
+func ixFindByTag(b *testing.B, org uint32, q int) {
series, err := ix.FindByTag(org, tagQueries[q].Expressions, 0)
if err != nil {
panic(err)
@@ -781,7 +781,7 @@ func BenchmarkTagFindSimpleIntersect(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
q := n % 2
- org := (n % 2) + 1
+ org := uint32((n % 2) + 1)
ixFindByTag(b, org, q)
}
}
@@ -792,7 +792,7 @@ func BenchmarkTagFindRegexIntersect(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
q := (n % 2) + 2
- org := (n % 2) + 1
+ org := uint32((n % 2) + 1)
ixFindByTag(b, org, q)
}
}
@@ -803,7 +803,7 @@ func BenchmarkTagFindMatchingAndFiltering(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
q := (n % 2) + 4
- org := (n % 2) + 1
+ org := uint32((n % 2) + 1)
ixFindByTag(b, org, q)
}
}
@@ -814,7 +814,7 @@ func BenchmarkTagFindMatchingAndFilteringWithRegex(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
q := (n % 2) + 6
- org := (n % 2) + 1
+ org := uint32((n % 2) + 1)
ixFindByTag(b, org, q)
}
}
diff --git a/idx/memory/memory_test.go b/idx/memory/memory_test.go
index 8b5f0d4c74..c66640a447 100644
--- a/idx/memory/memory_test.go
+++ b/idx/memory/memory_test.go
@@ -46,14 +46,14 @@ func getRandomString(n int, alphabets ...byte) string {
}
// getMetricData returns a count-length slice of MetricData's with random Name and the given org id
-func getMetricData(orgId, depth, count, interval int, prefix string, tagged bool) []*schema.MetricData {
+func getMetricData(orgId uint32, depth, count, interval int, prefix string, tagged bool) []*schema.MetricData {
data := make([]*schema.MetricData, count)
series := getSeriesNames(depth, count, prefix)
for i, s := range series {
data[i] = &schema.MetricData{
Name: s,
- OrgId: orgId,
+ OrgId: int(orgId),
Interval: interval,
}
if tagged {
@@ -96,7 +96,7 @@ func testGetAddKey(t *testing.T) {
org2Series := getMetricData(2, 2, 5, 10, "metric.org2", false)
for _, series := range [][]*schema.MetricData{publicSeries, org1Series, org2Series} {
- orgId := series[0].OrgId
+ orgId := uint32(series[0].OrgId)
Convey(fmt.Sprintf("When indexing metrics for orgId %d", orgId), t, func() {
for _, s := range series {
mkey, _ := schema.MKeyFromString(s.Id)
diff --git a/metrictank.go b/metrictank.go
index 752319d1f4..061d0edc2f 100644
--- a/metrictank.go
+++ b/metrictank.go
@@ -329,7 +329,7 @@ func main() {
log.Fatal(4, "public-org cannot be <0")
}
- idx.OrgIdPublic = *publicOrg
+ idx.OrgIdPublic = uint32(*publicOrg)
if memory.Enabled {
if metricIndex != nil {