Skip to content
This repository was archived by the owner on Aug 23, 2023. It is now read-only.

Commit 88830e3

Browse files
DieterbeDieterbe
authored
Merge pull request #1265 from grafana/track-findcache-entries
track number of entries in findCache.
2 parents 1d55764 + 206c9bf commit 88830e3

File tree

3 files changed

+253
-12
lines changed

3 files changed

+253
-12
lines changed

‎dashboards/main/metrictank.json

+220-11
Original file line numberDiff line numberDiff line change
@@ -3417,13 +3417,222 @@
34173417
"alignLevel": null
34183418
}
34193419
},
3420+
{
3421+
"aliasColors": {
3422+
"entries": "#1f78c1"
3423+
},
3424+
"bars": false,
3425+
"dashLength": 10,
3426+
"dashes": false,
3427+
"datasource": "$datasource",
3428+
"fill": 0,
3429+
"gridPos": {
3430+
"h": 4,
3431+
"w": 12,
3432+
"x": 0,
3433+
"y": 66
3434+
},
3435+
"id": 56,
3436+
"legend": {
3437+
"avg": false,
3438+
"current": false,
3439+
"max": false,
3440+
"min": false,
3441+
"show": true,
3442+
"total": false,
3443+
"values": false
3444+
},
3445+
"lines": true,
3446+
"linewidth": 1,
3447+
"links": [],
3448+
"nullPointMode": "null",
3449+
"percentage": false,
3450+
"pointradius": 5,
3451+
"points": false,
3452+
"renderer": "flot",
3453+
"seriesOverrides": [
3454+
{
3455+
"alias": "backoff",
3456+
"fill": 6,
3457+
"linewidth": 0,
3458+
"yaxis": 2
3459+
}
3460+
],
3461+
"spaceLength": 10,
3462+
"stack": false,
3463+
"steppedLine": false,
3464+
"targets": [
3465+
{
3466+
"refCount": 0,
3467+
"refId": "B",
3468+
"target": "alias(metrictank.stats.$environment.$instance.idx.memory.find-cache.entries.gauge32, 'entries')"
3469+
},
3470+
{
3471+
"refCount": 0,
3472+
"refId": "C",
3473+
"target": "alias(metrictank.stats.$environment.$instance.idx.memory.find-cache.backoff.gauge32, 'backoff')"
3474+
}
3475+
],
3476+
"thresholds": [],
3477+
"timeFrom": null,
3478+
"timeRegions": [],
3479+
"timeShift": null,
3480+
"title": "find cache",
3481+
"tooltip": {
3482+
"shared": true,
3483+
"sort": 0,
3484+
"value_type": "individual"
3485+
},
3486+
"type": "graph",
3487+
"xaxis": {
3488+
"buckets": null,
3489+
"mode": "time",
3490+
"name": null,
3491+
"show": true,
3492+
"values": []
3493+
},
3494+
"yaxes": [
3495+
{
3496+
"format": "short",
3497+
"label": null,
3498+
"logBase": 1,
3499+
"max": null,
3500+
"min": null,
3501+
"show": true
3502+
},
3503+
{
3504+
"format": "short",
3505+
"label": null,
3506+
"logBase": 1,
3507+
"max": null,
3508+
"min": "0",
3509+
"show": true
3510+
}
3511+
],
3512+
"yaxis": {
3513+
"align": false,
3514+
"alignLevel": null
3515+
}
3516+
},
3517+
{
3518+
"aliasColors": {
3519+
"invalidation.drop": "#f29191",
3520+
"invalidation.exec": "#5195ce",
3521+
"invalidation.recv": "#ba43a9",
3522+
"miss": "#e0752d"
3523+
},
3524+
"bars": false,
3525+
"dashLength": 10,
3526+
"dashes": false,
3527+
"datasource": "$datasource",
3528+
"fill": 1,
3529+
"gridPos": {
3530+
"h": 4,
3531+
"w": 12,
3532+
"x": 12,
3533+
"y": 66
3534+
},
3535+
"id": 57,
3536+
"legend": {
3537+
"avg": false,
3538+
"current": false,
3539+
"max": false,
3540+
"min": false,
3541+
"show": true,
3542+
"total": false,
3543+
"values": false
3544+
},
3545+
"lines": true,
3546+
"linewidth": 1,
3547+
"links": [],
3548+
"nullPointMode": "null",
3549+
"percentage": false,
3550+
"pointradius": 5,
3551+
"points": false,
3552+
"renderer": "flot",
3553+
"seriesOverrides": [
3554+
{
3555+
"alias": "invalidation.recv",
3556+
"yaxis": 2
3557+
},
3558+
{
3559+
"alias": "invalidation.exec",
3560+
"yaxis": 2
3561+
},
3562+
{
3563+
"alias": "invalidation.drop",
3564+
"yaxis": 2
3565+
},
3566+
{
3567+
"alias": "/invalidation/",
3568+
"fill": 3,
3569+
"linewidth": 0,
3570+
"stack": "A"
3571+
}
3572+
],
3573+
"spaceLength": 10,
3574+
"stack": false,
3575+
"steppedLine": false,
3576+
"targets": [
3577+
{
3578+
"refCount": 0,
3579+
"refId": "A",
3580+
"target": "groupByNode(metrictank.stats.$environment.$instance.idx.memory.find-cache.ops.*.rate32, 8, 'sum')"
3581+
},
3582+
{
3583+
"refCount": 0,
3584+
"refId": "D",
3585+
"target": "groupByNodes(metrictank.stats.$environment.$instance.idx.memory.find-cache.invalidation.*.rate32, 'sum', 7, 8)"
3586+
}
3587+
],
3588+
"thresholds": [],
3589+
"timeFrom": null,
3590+
"timeRegions": [],
3591+
"timeShift": null,
3592+
"title": "find cache",
3593+
"tooltip": {
3594+
"shared": true,
3595+
"sort": 0,
3596+
"value_type": "individual"
3597+
},
3598+
"type": "graph",
3599+
"xaxis": {
3600+
"buckets": null,
3601+
"mode": "time",
3602+
"name": null,
3603+
"show": true,
3604+
"values": []
3605+
},
3606+
"yaxes": [
3607+
{
3608+
"format": "short",
3609+
"label": null,
3610+
"logBase": 1,
3611+
"max": null,
3612+
"min": null,
3613+
"show": true
3614+
},
3615+
{
3616+
"format": "short",
3617+
"label": null,
3618+
"logBase": 1,
3619+
"max": null,
3620+
"min": null,
3621+
"show": true
3622+
}
3623+
],
3624+
"yaxis": {
3625+
"align": false,
3626+
"alignLevel": null
3627+
}
3628+
},
34203629
{
34213630
"collapsed": false,
34223631
"gridPos": {
34233632
"h": 1,
34243633
"w": 24,
34253634
"x": 0,
3426-
"y": 66
3635+
"y": 70
34273636
},
34283637
"id": 46,
34293638
"panels": [],
@@ -3445,7 +3654,7 @@
34453654
"h": 7,
34463655
"w": 8,
34473656
"x": 0,
3448-
"y": 67
3657+
"y": 71
34493658
},
34503659
"id": 24,
34513660
"legend": {
@@ -3534,7 +3743,7 @@
35343743
"h": 7,
35353744
"w": 8,
35363745
"x": 8,
3537-
"y": 67
3746+
"y": 71
35383747
},
35393748
"id": 25,
35403749
"legend": {
@@ -3638,7 +3847,7 @@
36383847
"h": 7,
36393848
"w": 8,
36403849
"x": 16,
3641-
"y": 67
3850+
"y": 71
36423851
},
36433852
"id": 26,
36443853
"legend": {
@@ -3720,7 +3929,7 @@
37203929
"h": 1,
37213930
"w": 24,
37223931
"x": 0,
3723-
"y": 74
3932+
"y": 78
37243933
},
37253934
"id": 47,
37263935
"panels": [],
@@ -3745,7 +3954,7 @@
37453954
"h": 6,
37463955
"w": 8,
37473956
"x": 0,
3748-
"y": 75
3957+
"y": 79
37493958
},
37503959
"id": 27,
37513960
"legend": {
@@ -3876,7 +4085,7 @@
38764085
"h": 6,
38774086
"w": 7,
38784087
"x": 8,
3879-
"y": 75
4088+
"y": 79
38804089
},
38814090
"id": 50,
38824091
"legend": {
@@ -4012,7 +4221,7 @@
40124221
"h": 6,
40134222
"w": 9,
40144223
"x": 15,
4015-
"y": 75
4224+
"y": 79
40164225
},
40174226
"id": 28,
40184227
"legend": {
@@ -4123,7 +4332,7 @@
41234332
"h": 1,
41244333
"w": 24,
41254334
"x": 0,
4126-
"y": 81
4335+
"y": 85
41274336
},
41284337
"id": 48,
41294338
"panels": [],
@@ -4150,7 +4359,7 @@
41504359
"h": 7,
41514360
"w": 12,
41524361
"x": 0,
4153-
"y": 82
4362+
"y": 86
41544363
},
41554364
"id": 32,
41564365
"legend": {
@@ -4273,7 +4482,7 @@
42734482
"h": 7,
42744483
"w": 12,
42754484
"x": 12,
4276-
"y": 82
4485+
"y": 86
42774486
},
42784487
"id": 15,
42794488
"legend": {

‎docs/metrics.md

+2
Original file line numberDiff line numberDiff line change
@@ -186,6 +186,8 @@ number of series that have been excluded from responses due to their lastUpdate
186186
the duration of memory idx find
187187
* `idx.memory.find-cache.backoff`:
188188
the number of find caches in backoff mode
189+
* `idx.memory.find-cache.entries`:
190+
the number of entries in the cache
189191
* `idx.memory.find-cache.invalidation.drop`:
190192
the number of dropped invalidation requests
191193
* `idx.memory.find-cache.invalidation.exec`:

‎idx/memory/find_cache.go

+31-1
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ var (
1818
findCacheMiss = stats.NewCounterRate32("idx.memory.find-cache.ops.miss")
1919
// metric idx.memory.find-cache.backoff is the number of find caches in backoff mode
2020
findCacheBackoff = stats.NewGauge32("idx.memory.find-cache.backoff")
21+
// metric idx.memory.find-cache.entries is the number of entries in the cache
22+
findCacheEntries = stats.NewGauge32("idx.memory.find-cache.entries")
2123
// metric idx.memory.find-cache.invalidation.recv is the number of received invalidation requests
2224
findCacheInvalidationsReceived = stats.NewCounterRate32("idx.memory.find-cache.invalidation.recv")
2325
// metric idx.memory.find-cache.invalidation.exec is the number of executed invalidation requests
@@ -74,6 +76,7 @@ func NewFindCache(size, invalidateQueueSize, invalidateMaxSize int, invalidateMa
7476
cache: make(map[uint32]*lru.Cache),
7577
}
7678
go fc.processInvalidateQueue()
79+
go fc.stats()
7780
return fc
7881
}
7982

@@ -177,7 +180,6 @@ func (c *FindCache) InvalidateFor(orgId uint32, path string) {
177180
case c.invalidateReqs <- req:
178181
default:
179182
c.triggerBackoff()
180-
return
181183
}
182184
}
183185

@@ -213,6 +215,34 @@ func (c *FindCache) Shutdown() {
213215
c.shutdown <- struct{}{}
214216
}
215217

218+
func (c *FindCache) stats() {
219+
var prev int
220+
tick := time.NewTicker(2 * time.Second)
221+
for {
222+
select {
223+
case <-tick.C:
224+
size := 0
225+
c.RLock()
226+
for _, cache := range c.cache {
227+
size += cache.Len()
228+
}
229+
c.RUnlock()
230+
// we track and subtract our previous entry, so that this works for
231+
// partitioned index, which has multiple findcaches. want to see
232+
// total entries across them all.
233+
if size > prev {
234+
findCacheEntries.AddUint32(uint32(size - prev))
235+
} else {
236+
findCacheEntries.DecUint32(uint32(prev - size))
237+
}
238+
prev = size
239+
case <-c.shutdown:
240+
tick.Stop()
241+
return
242+
}
243+
}
244+
}
245+
216246
func (c *FindCache) processInvalidateQueue() {
217247
type invalidateBuffer struct {
218248
buffer map[uint32][]invalidateRequest

0 commit comments

Comments
 (0)