[metrics] add lru evict atime metric

cache/disk/disk.go

@@ -20,6 +20,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/buchgr/bazel-remote/cache"
 	"github.com/buchgr/bazel-remote/cache/disk/casblob"
@@ -142,7 +143,7 @@ func New(dir string, maxSizeBytes int64, opts ...Option) (Cache, error) {
 	// The eviction callback deletes the file from disk.
 	// This function is only called while the lock is held
 	// by the current goroutine.
-	onEvict := func(key Key, value lruItem) {
+	onEvict := func(key Key, value lruItem) time.Time {
 		ks := key.(string)
 		hash := ks[len(ks)-sha256.Size*2:]
 		var kind cache.EntryKind = cache.AC
@@ -156,8 +157,15 @@ func New(dir string, maxSizeBytes int64, opts ...Option) (Cache, error) {
 
 		f := filepath.Join(dir, c.FileLocation(kind, value.legacy, hash, value.size, value.random))
 
+		ts, err := atime.Stat(f)
+		if err != nil {
+			ts = time.Time{}
+		}
+
 		// Run in a goroutine so we can release the lock sooner.
 		go c.removeFile(f)
+
+		return ts
 	}
 
 	c.lru = NewSizedLRU(maxSizeBytes, onEvict)

cache/disk/disk_test.go

@@ -430,9 +430,10 @@ func TestCacheExistingFiles(t *testing.T) {
 
 	evicted := []Key{}
 	origOnEvict := testCache.lru.onEvict
-	testCache.lru.onEvict = func(key Key, value lruItem) {
+	testCache.lru.onEvict = func(key Key, value lruItem) time.Time {
 		evicted = append(evicted, key.(string))
 		origOnEvict(key, value)
+		return time.Time{}
 	}
 
 	if testCache.lru.Len() != 4 {

cache/disk/lru.go

@@ -4,6 +4,7 @@ import (
 	"container/list"
 	"errors"
 	"fmt"
+	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
 )
@@ -17,7 +18,7 @@ import (
 type Key interface{}
 
 // EvictCallback is the type of callbacks that are invoked when items are evicted.
-type EvictCallback func(key Key, value lruItem)
+type EvictCallback func(key Key, value lruItem) time.Time
 
 // SizedLRU is an LRU cache that will keep its total size below maxSize by evicting
 // items.
@@ -44,10 +45,11 @@ type SizedLRU struct {
 
 	onEvict EvictCallback
 
-	gaugeCacheSizeBytes     prometheus.Gauge
-	gaugeCacheLogicalBytes  prometheus.Gauge
-	counterEvictedBytes     prometheus.Counter
-	counterOverwrittenBytes prometheus.Counter
+	gaugeCacheLastEvictAtime prometheus.Gauge
+	gaugeCacheSizeBytes      prometheus.Gauge
+	gaugeCacheLogicalBytes   prometheus.Gauge
+	counterEvictedBytes      prometheus.Counter
+	counterOverwrittenBytes  prometheus.Counter
 }
 
 type entry struct {
@@ -67,6 +69,10 @@ func NewSizedLRU(maxSize int64, onEvict EvictCallback) SizedLRU {
 		cache:   make(map[interface{}]*list.Element),
 		onEvict: onEvict,
 
+		gaugeCacheLastEvictAtime: prometheus.NewGauge(prometheus.GaugeOpts{
+			Name: "bazel_remote_disk_cache_last_evicted_atime",
+			Help: "The access time of the most recently evicted item from the LRU cache.",
+		}),
 		gaugeCacheSizeBytes: prometheus.NewGauge(prometheus.GaugeOpts{
 			Name: "bazel_remote_disk_cache_size_bytes",
 			Help: "The current number of bytes in the disk backend",
@@ -87,6 +93,7 @@ func NewSizedLRU(maxSize int64, onEvict EvictCallback) SizedLRU {
 }
 
 func (c *SizedLRU) RegisterMetrics() {
+	prometheus.MustRegister(c.gaugeCacheLastEvictAtime)
 	prometheus.MustRegister(c.gaugeCacheSizeBytes)
 	prometheus.MustRegister(c.gaugeCacheLogicalBytes)
 	prometheus.MustRegister(c.counterEvictedBytes)
@@ -272,7 +279,11 @@ func (c *SizedLRU) removeElement(e *list.Element) {
 	c.counterEvictedBytes.Add(float64(kv.value.sizeOnDisk))
 
 	if c.onEvict != nil {
-		c.onEvict(kv.key, kv.value)
+		ts := c.onEvict(kv.key, kv.value)
+
+		if !ts.IsZero() {
+			c.gaugeCacheLastEvictAtime.Set(float64(ts.Unix()))
+		}
 	}
 }
 

cache/disk/lru_test.go

@@ -4,6 +4,7 @@ import (
 	"math"
 	"reflect"
 	"testing"
+	"time"
 )
 
 func checkSizeAndNumItems(t *testing.T, lru SizedLRU, expSize int64, expNum int) {
@@ -60,8 +61,9 @@ func TestBasics(t *testing.T) {
 func TestEviction(t *testing.T) {
 	// Keep track of evictions using the callback
 	var evictions []int
-	onEvict := func(key Key, value lruItem) {
+	onEvict := func(key Key, value lruItem) time.Time {
 		evictions = append(evictions, key.(int))
+		return time.Time{}
 	}
 
 	lru := NewSizedLRU(10*BlockSize, onEvict)