Skip to content
This repository was archived by the owner on Aug 23, 2023. It is now read-only.

Commit 4e2cf41

Browse files
DieterbeDieterbe
authored
Merge pull request #880 from grafana/explicit-orgidpublic
public orgId -1 -> 0 and related cleanups
2 parents a7ebc62 + 5db1111 commit 4e2cf41

File tree

7 files changed

+50
-74
lines changed

7 files changed

+50
-74
lines changed

‎docs/http-api.md

+3-4
Original file line numberDiff line numberDiff line change
@@ -35,8 +35,7 @@ POST /metrics/index.json
3535

3636
* header `X-Org-Id` required
3737

38-
Returns metrics stored under the given org, as well as public data under org -1 (see [multi-tenancy](https://github.com/grafana/metrictank/blob/master/docs/multi-tenancy.md))
39-
If orgId is -1, returns the metrics for all orgs. (but you can't necessarily distinguish which org a metric is from)
38+
Returns metrics stored under the given org, as well as public data under org 0 (see [multi-tenancy](https://github.com/grafana/metrictank/blob/master/docs/multi-tenancy.md))
4039

4140
#### Example
4241

@@ -57,7 +56,7 @@ POST /metrics/find
5756
* format: json, treejson, completer, pickle, or msgpack. (defaults to json)
5857
* jsonp
5958

60-
Returns metrics which match the query and are stored under the given org or are public data under org -1 (see [multi-tenancy](https://github.com/grafana/metrictank/blob/master/docs/multi-tenancy.md))
59+
Returns metrics which match the query and are stored under the given org or are public data under org 0 (see [multi-tenancy](https://github.com/grafana/metrictank/blob/master/docs/multi-tenancy.md))
6160
the completer format is for completion UI's such as graphite-web.
6261
json and treejson are the same.
6362

@@ -113,7 +112,7 @@ POST /render
113112

114113
If metrictank doesn't have a requested function, it always proxies to graphite, irrespective of this setting.
115114

116-
Data queried for must be stored under the given org or be public data under org -1 (see [multi-tenancy](https://github.com/grafana/metrictank/blob/master/docs/multi-tenancy.md))
115+
Data queried for must be stored under the given org or be public data under org 0 (see [multi-tenancy](https://github.com/grafana/metrictank/blob/master/docs/multi-tenancy.md))
117116

118117
#### Example
119118

‎docs/multi-tenancy.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -10,4 +10,4 @@ A metrictank based stack is multi-tenant. Here's how it works:
1010
* For a secure setup, you must make sure these headers cannot be specified by users. You may need to run something in front to set the header correctly after authentication
1111
(e.g. [tsdb-gw](https://github.com/raintank/tsdb-gw)
1212
* orgs can only see the data that lives under their org-id, and also public data
13-
* public data is stored under orgId -1 and is visible to everyone.
13+
* public data is stored under orgId 0 and is visible to everyone.

‎idx/cassandra/cassandra.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -492,9 +492,9 @@ func (c *CasIdx) deleteDef(def *idx.Archive) error {
492492
return fmt.Errorf("unable to delete metricDef %s from index after %d attempts.", def.Id, attempts)
493493
}
494494

495-
func (c *CasIdx) Prune(orgId int, oldest time.Time) ([]idx.Archive, error) {
495+
func (c *CasIdx) Prune(oldest time.Time) ([]idx.Archive, error) {
496496
pre := time.Now()
497-
pruned, err := c.MemoryIdx.Prune(orgId, oldest)
497+
pruned, err := c.MemoryIdx.Prune(oldest)
498498
statPruneDuration.Value(time.Since(pre))
499499
return pruned, err
500500
}
@@ -504,7 +504,7 @@ func (c *CasIdx) prune() {
504504
for range ticker.C {
505505
log.Debug("cassandra-idx: pruning items from index that have not been seen for %s", maxStale.String())
506506
staleTs := time.Now().Add(maxStale * -1)
507-
_, err := c.Prune(-1, staleTs)
507+
_, err := c.Prune(staleTs)
508508
if err != nil {
509509
log.Error(3, "cassandra-idx: prune error. %s", err)
510510
}

‎idx/cassandra/cassandra_test.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,7 @@ func TestGetAddKey(t *testing.T) {
125125
ix := New()
126126
initForTests(ix)
127127

128-
publicSeries := getMetricData(-1, 2, 5, 10, "metric.public")
128+
publicSeries := getMetricData(idx.OrgIdPublic, 2, 5, 10, "metric.public")
129129
org1Series := getMetricData(1, 2, 5, 10, "metric.org1")
130130
org2Series := getMetricData(2, 2, 5, 10, "metric.org2")
131131

@@ -138,7 +138,7 @@ func TestGetAddKey(t *testing.T) {
138138
Convey(fmt.Sprintf("Then listing metrics for OrgId %d", orgId), func() {
139139
defs := ix.List(orgId)
140140
numSeries := len(series)
141-
if orgId != -1 {
141+
if orgId != idx.OrgIdPublic {
142142
numSeries += 5
143143
}
144144
So(defs, ShouldHaveLength, numSeries)
@@ -258,7 +258,7 @@ func TestAddToWriteQueue(t *testing.T) {
258258
func TestFind(t *testing.T) {
259259
ix := New()
260260
initForTests(ix)
261-
for _, s := range getMetricData(-1, 2, 5, 10, "metric.demo") {
261+
for _, s := range getMetricData(idx.OrgIdPublic, 2, 5, 10, "metric.demo") {
262262
ix.AddOrUpdate(s, 1)
263263
}
264264
for _, s := range getMetricData(1, 2, 5, 10, "metric.demo") {

‎idx/idx.go

+6-6
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@ import (
1313
schema "gopkg.in/raintank/schema.v1"
1414
)
1515

16+
const OrgIdPublic = 0
17+
1618
var (
1719
BothBranchAndLeaf = errors.New("node can't be both branch and leaf")
1820
BranchUnderLeaf = errors.New("can't add branch under leaf")
@@ -118,20 +120,18 @@ type MetricIndex interface {
118120

119121
// Find searches the index. The method is passed an OrgId, a query
120122
// pattern and a unix timestamp. Searches should return all nodes that match for
121-
// the given OrgId and OrgId -1. The pattern should be handled in the same way
123+
// the given OrgId and OrgIdPublic. The pattern should be handled in the same way
122124
// Graphite would. see https://graphite.readthedocs.io/en/latest/render_api.html#paths-and-wildcards
123125
// And the unix stimestamp is used to ignore series that have been stale since
124126
// the timestamp.
125127
Find(int, string, int64) ([]Node, error)
126128

127-
// List returns all Archives for the passed OrgId, or for all organisations if -1 is provided.
129+
// List returns all Archives for the passed OrgId and the public orgId
128130
List(int) []Archive
129131

130-
// Prune deletes all metrics from the index for the passed org where
131-
// the last time the metric was seen is older then the passed timestamp. If the org
132-
// passed is -1, then the all orgs should be examined for stale metrics to be deleted.
132+
// Prune deletes all metrics that haven't been seen since the given timestamp.
133133
// It returns all Archives deleted and any error encountered.
134-
Prune(int, time.Time) ([]Archive, error)
134+
Prune(time.Time) ([]Archive, error)
135135

136136
// FindByTag takes a list of expressions in the format key<operator>value.
137137
// The allowed operators are: =, !=, =~, !=~.

‎idx/memory/memory.go

+19-42
Original file line numberDiff line numberDiff line change
@@ -836,15 +836,17 @@ func (m *MemoryIdx) Find(orgId int, pattern string, from int64) ([]idx.Node, err
836836
if err != nil {
837837
return nil, err
838838
}
839-
publicNodes, err := m.find(-1, pattern)
840-
if err != nil {
841-
return nil, err
839+
if orgId != idx.OrgIdPublic {
840+
publicNodes, err := m.find(idx.OrgIdPublic, pattern)
841+
if err != nil {
842+
return nil, err
843+
}
844+
matchedNodes = append(matchedNodes, publicNodes...)
842845
}
843-
matchedNodes = append(matchedNodes, publicNodes...)
844846
log.Debug("memory-idx: %d nodes matching pattern %s found", len(matchedNodes), pattern)
845847
results := make([]idx.Node, 0)
846848
seen := make(map[string]struct{})
847-
// if there are public (orgId -1) and private leaf nodes with the same series
849+
// if there are public (orgId OrgIdPublic) and private leaf nodes with the same series
848850
// path, then the public metricDefs will be excluded.
849851
for _, n := range matchedNodes {
850852
if _, ok := seen[n.Path]; !ok {
@@ -977,26 +979,11 @@ func (m *MemoryIdx) List(orgId int) []idx.Archive {
977979
m.RLock()
978980
defer m.RUnlock()
979981

980-
orgs := make(map[int]struct{})
981-
if orgId == -1 {
982-
log.Info("memory-idx: returning all metricDefs for all orgs")
983-
for org := range m.tree {
984-
orgs[org] = struct{}{}
985-
}
986-
for org := range m.tags {
987-
orgs[org] = struct{}{}
988-
}
989-
} else {
990-
orgs[-1] = struct{}{}
991-
orgs[orgId] = struct{}{}
992-
}
993-
994982
defs := make([]idx.Archive, 0)
995983
for _, def := range m.defById {
996-
if _, ok := orgs[def.OrgId]; !ok {
997-
continue
984+
if def.OrgId == orgId || def.OrgId == idx.OrgIdPublic {
985+
defs = append(defs, *def)
998986
}
999-
defs = append(defs, *def)
1000987
}
1001988

1002989
statListDuration.Value(time.Since(pre))
@@ -1187,24 +1174,20 @@ func (m *MemoryIdx) delete(orgId int, n *Node, deleteEmptyParents, deleteChildre
11871174
}
11881175

11891176
// delete series from the index if they have not been seen since "oldest"
1190-
func (m *MemoryIdx) Prune(orgId int, oldest time.Time) ([]idx.Archive, error) {
1177+
func (m *MemoryIdx) Prune(oldest time.Time) ([]idx.Archive, error) {
11911178
oldestUnix := oldest.Unix()
11921179
orgs := make(map[int]struct{})
1193-
if orgId == -1 {
1194-
log.Info("memory-idx: pruning stale metricDefs across all orgs")
1195-
m.RLock()
1196-
for org := range m.tree {
1180+
log.Info("memory-idx: pruning stale metricDefs across all orgs")
1181+
m.RLock()
1182+
for org := range m.tree {
1183+
orgs[org] = struct{}{}
1184+
}
1185+
if TagSupport {
1186+
for org := range m.tags {
11971187
orgs[org] = struct{}{}
11981188
}
1199-
if TagSupport {
1200-
for org := range m.tags {
1201-
orgs[org] = struct{}{}
1202-
}
1203-
}
1204-
m.RUnlock()
1205-
} else {
1206-
orgs[orgId] = struct{}{}
12071189
}
1190+
m.RUnlock()
12081191

12091192
var pruned []idx.Archive
12101193
toPruneUntagged := make(map[int]map[string]struct{}, len(orgs))
@@ -1218,10 +1201,6 @@ func (m *MemoryIdx) Prune(orgId int, oldest time.Time) ([]idx.Archive, error) {
12181201
m.RLock()
12191202
DEFS:
12201203
for _, def := range m.defById {
1221-
if _, ok := orgs[def.OrgId]; !ok {
1222-
continue DEFS
1223-
}
1224-
12251204
if def.LastUpdate >= oldestUnix {
12261205
continue DEFS
12271206
}
@@ -1300,9 +1279,7 @@ DEFS:
13001279

13011280
statMetricsActive.Add(-1 * len(pruned))
13021281

1303-
if orgId == -1 {
1304-
log.Info("memory-idx: pruning stale metricDefs from memory for all orgs took %s", time.Since(pre).String())
1305-
}
1282+
log.Info("memory-idx: pruning stale metricDefs from memory for all orgs took %s", time.Since(pre).String())
13061283

13071284
statPruneDuration.Value(time.Since(pre))
13081285
return pruned, nil

‎idx/memory/memory_test.go

+15-15
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ func testGetAddKey(t *testing.T) {
8686
ix := New()
8787
ix.Init()
8888

89-
publicSeries := getMetricData(-1, 2, 5, 10, "metric.public", false)
89+
publicSeries := getMetricData(idx.OrgIdPublic, 2, 5, 10, "metric.public", false)
9090
org1Series := getMetricData(1, 2, 5, 10, "metric.org1", false)
9191
org2Series := getMetricData(2, 2, 5, 10, "metric.org2", false)
9292

@@ -99,7 +99,7 @@ func testGetAddKey(t *testing.T) {
9999
Convey(fmt.Sprintf("Then listing metrics for OrgId %d", orgId), func() {
100100
defs := ix.List(orgId)
101101
numSeries := len(series)
102-
if orgId != -1 {
102+
if orgId != idx.OrgIdPublic {
103103
numSeries += 5
104104
}
105105
So(defs, ShouldHaveLength, numSeries)
@@ -128,7 +128,7 @@ func TestFind(t *testing.T) {
128128
func testFind(t *testing.T) {
129129
ix := New()
130130
ix.Init()
131-
for _, s := range getMetricData(-1, 2, 5, 10, "metric.demo", false) {
131+
for _, s := range getMetricData(idx.OrgIdPublic, 2, 5, 10, "metric.demo", false) {
132132
s.Time = 10 * 86400
133133
ix.AddOrUpdate(s, 1)
134134
}
@@ -256,7 +256,7 @@ func testDelete(t *testing.T) {
256256
ix := New()
257257
ix.Init()
258258

259-
publicSeries := getMetricData(-1, 2, 5, 10, "metric.public", false)
259+
publicSeries := getMetricData(idx.OrgIdPublic, 2, 5, 10, "metric.public", false)
260260
org1Series := getMetricData(1, 2, 5, 10, "metric.org1", false)
261261

262262
for _, s := range publicSeries {
@@ -271,7 +271,7 @@ func TestDeleteTagged(t *testing.T) {
271271
ix := New()
272272
ix.Init()
273273

274-
publicSeries := getMetricData(-1, 2, 5, 10, "metric.public", true)
274+
publicSeries := getMetricData(idx.OrgIdPublic, 2, 5, 10, "metric.public", true)
275275
org1Series := getMetricData(1, 2, 5, 10, "metric.org1", true)
276276

277277
for _, s := range publicSeries {
@@ -506,12 +506,12 @@ func TestPruneTaggedSeries(t *testing.T) {
506506
}
507507

508508
Convey("after populating index", t, func() {
509-
defs := ix.List(-1)
509+
defs := ix.List(1)
510510
So(defs, ShouldHaveLength, 10)
511511
})
512512

513513
Convey("When purging old series", t, func() {
514-
purged, err := ix.Prune(1, time.Unix(2, 0))
514+
purged, err := ix.Prune(time.Unix(2, 0))
515515
So(err, ShouldBeNil)
516516
So(purged, ShouldHaveLength, 5)
517517
nodes, err := ix.FindByTag(1, []string{"name=~metric\\.bah.*", "series_id=~[0-4]"}, 0)
@@ -523,7 +523,7 @@ func TestPruneTaggedSeries(t *testing.T) {
523523
})
524524

525525
Convey("after purge", t, func() {
526-
defs := ix.List(-1)
526+
defs := ix.List(1)
527527
So(defs, ShouldHaveLength, 5)
528528
data := &schema.MetricData{
529529
Name: defs[0].Name,
@@ -538,7 +538,7 @@ func TestPruneTaggedSeries(t *testing.T) {
538538
data.SetId()
539539
ix.AddOrUpdate(data, 1)
540540
Convey("When purging old series", func() {
541-
purged, err := ix.Prune(1, time.Unix(12, 0))
541+
purged, err := ix.Prune(time.Unix(12, 0))
542542
So(err, ShouldBeNil)
543543
So(purged, ShouldHaveLength, 4)
544544
nodes, err := ix.FindByTag(1, []string{"name=~metric\\.foo.*", "series_id=~[0-4]"}, 0)
@@ -581,7 +581,7 @@ func TestPruneTaggedSeriesWithCollidingTagSets(t *testing.T) {
581581
}
582582

583583
Convey("When purging old series", t, func() {
584-
purged, err := ix.Prune(1, time.Unix(2, 0))
584+
purged, err := ix.Prune(time.Unix(2, 0))
585585
So(err, ShouldBeNil)
586586
So(purged, ShouldHaveLength, 0)
587587
})
@@ -593,7 +593,7 @@ func TestPruneTaggedSeriesWithCollidingTagSets(t *testing.T) {
593593
})
594594

595595
Convey("When purging newer series", t, func() {
596-
purged, err := ix.Prune(1, time.Unix(20, 0))
596+
purged, err := ix.Prune(time.Unix(20, 0))
597597
So(err, ShouldBeNil)
598598
So(purged, ShouldHaveLength, 2)
599599
})
@@ -638,11 +638,11 @@ func testPrune(t *testing.T) {
638638
ix.AddOrUpdate(d, 1)
639639
}
640640
Convey("after populating index", t, func() {
641-
defs := ix.List(-1)
641+
defs := ix.List(1)
642642
So(defs, ShouldHaveLength, 10)
643643
})
644644
Convey("When purging old series", t, func() {
645-
purged, err := ix.Prune(1, time.Unix(2, 0))
645+
purged, err := ix.Prune(time.Unix(2, 0))
646646
So(err, ShouldBeNil)
647647
So(purged, ShouldHaveLength, 5)
648648
nodes, err := ix.Find(1, "metric.bah.*", 0)
@@ -654,7 +654,7 @@ func testPrune(t *testing.T) {
654654

655655
})
656656
Convey("after purge", t, func() {
657-
defs := ix.List(-1)
657+
defs := ix.List(1)
658658
So(defs, ShouldHaveLength, 5)
659659
data := &schema.MetricData{
660660
Name: defs[0].Name,
@@ -667,7 +667,7 @@ func testPrune(t *testing.T) {
667667
data.SetId()
668668
ix.AddOrUpdate(data, 0)
669669
Convey("When purging old series", func() {
670-
purged, err := ix.Prune(1, time.Unix(12, 0))
670+
purged, err := ix.Prune(time.Unix(12, 0))
671671
So(err, ShouldBeNil)
672672
So(purged, ShouldHaveLength, 4)
673673
nodes, err := ix.Find(1, "metric.foo.*", 0)

0 commit comments

Comments
 (0)