Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add latency histogram for fetching index cache #6749

Merged
merged 3 commits into from
Sep 26, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re

- [#6605](https://github.com/thanos-io/thanos/pull/6605) Query Frontend: Support vertical sharding binary expression with metric name when no matching labels specified.
- [#6308](https://github.com/thanos-io/thanos/pull/6308) Ruler: Support configuration flag that allows customizing template for alert message.
- [#6749](https://github.com/thanos-io/thanos/pull/6308) Store Gateway: Added `thanos_store_index_cache_fetch_duration_seconds` histogram for tracking latency of fetching data from index cache.

### Changed

Expand Down
6 changes: 6 additions & 0 deletions pkg/store/cache/cache.go
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ type commonMetrics struct {
requestTotal *prometheus.CounterVec
hitsTotal *prometheus.CounterVec
dataSizeBytes *prometheus.HistogramVec
fetchLatency *prometheus.HistogramVec
}

func newCommonMetrics(reg prometheus.Registerer) *commonMetrics {
Expand All @@ -80,6 +81,11 @@ func newCommonMetrics(reg prometheus.Registerer) *commonMetrics {
32, 256, 512, 1024, 32 * 1024, 256 * 1024, 512 * 1024, 1024 * 1024, 32 * 1024 * 1024, 64 * 1024 * 1024, 128 * 1024 * 1024, 256 * 1024 * 1024, 512 * 1024 * 1024,
},
}, []string{"item_type"}),
fetchLatency: promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{
Name: "thanos_store_index_cache_fetch_duration_seconds",
Help: "Histogram to track latency to fetch items from index cache",
Buckets: []float64{0.01, 0.1, 0.3, 0.6, 1, 3, 6, 10, 15, 20, 30, 45, 60, 90, 120},
}, []string{"item_type"}),
}
}

Expand Down
9 changes: 9 additions & 0 deletions pkg/store/cache/inmemory.go
Original file line number Diff line number Diff line change
Expand Up @@ -302,6 +302,9 @@ func (c *InMemoryIndexCache) StorePostings(blockID ulid.ULID, l labels.Label, v
// FetchMultiPostings fetches multiple postings - each identified by a label -
// and returns a map containing cache hits, along with a list of missing keys.
func (c *InMemoryIndexCache) FetchMultiPostings(_ context.Context, blockID ulid.ULID, keys []labels.Label) (hits map[labels.Label][]byte, misses []labels.Label) {
timer := prometheus.NewTimer(c.commonMetrics.fetchLatency.WithLabelValues(cacheTypePostings))
defer timer.ObserveDuration()

hits = map[labels.Label][]byte{}

blockIDKey := blockID.String()
Expand All @@ -325,6 +328,9 @@ func (c *InMemoryIndexCache) StoreExpandedPostings(blockID ulid.ULID, matchers [

// FetchExpandedPostings fetches expanded postings and returns cached data and a boolean value representing whether it is a cache hit or not.
func (c *InMemoryIndexCache) FetchExpandedPostings(_ context.Context, blockID ulid.ULID, matchers []*labels.Matcher) ([]byte, bool) {
timer := prometheus.NewTimer(c.commonMetrics.fetchLatency.WithLabelValues(cacheTypeExpandedPostings))
defer timer.ObserveDuration()

if b, ok := c.get(cacheTypeExpandedPostings, cacheKey{blockID.String(), cacheKeyExpandedPostings(labelMatchersToString(matchers)), ""}); ok {
return b, true
}
Expand All @@ -341,6 +347,9 @@ func (c *InMemoryIndexCache) StoreSeries(blockID ulid.ULID, id storage.SeriesRef
// FetchMultiSeries fetches multiple series - each identified by ID - from the cache
// and returns a map containing cache hits, along with a list of missing IDs.
func (c *InMemoryIndexCache) FetchMultiSeries(_ context.Context, blockID ulid.ULID, ids []storage.SeriesRef) (hits map[storage.SeriesRef][]byte, misses []storage.SeriesRef) {
timer := prometheus.NewTimer(c.commonMetrics.fetchLatency.WithLabelValues(cacheTypeSeries))
defer timer.ObserveDuration()

hits = map[storage.SeriesRef][]byte{}

blockIDKey := blockID.String()
Expand Down
34 changes: 25 additions & 9 deletions pkg/store/cache/memcached.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,15 +33,18 @@ type RemoteIndexCache struct {
compressionScheme string

// Metrics.
postingRequests prometheus.Counter
seriesRequests prometheus.Counter
expandedPostingRequests prometheus.Counter
postingHits prometheus.Counter
seriesHits prometheus.Counter
expandedPostingHits prometheus.Counter
postingDataSizeBytes prometheus.Observer
expandedPostingDataSizeBytes prometheus.Observer
seriesDataSizeBytes prometheus.Observer
postingRequests prometheus.Counter
seriesRequests prometheus.Counter
expandedPostingRequests prometheus.Counter
postingHits prometheus.Counter
seriesHits prometheus.Counter
expandedPostingHits prometheus.Counter
postingDataSizeBytes prometheus.Observer
expandedPostingDataSizeBytes prometheus.Observer
seriesDataSizeBytes prometheus.Observer
postingsFetchDuration prometheus.Observer
expandedPostingsFetchDuration prometheus.Observer
seriesFetchDuration prometheus.Observer
}

// NewRemoteIndexCache makes a new RemoteIndexCache.
Expand All @@ -68,6 +71,10 @@ func NewRemoteIndexCache(logger log.Logger, cacheClient cacheutil.RemoteCacheCli
c.seriesDataSizeBytes = commonMetrics.dataSizeBytes.WithLabelValues(cacheTypeSeries)
c.expandedPostingDataSizeBytes = commonMetrics.dataSizeBytes.WithLabelValues(cacheTypeExpandedPostings)

c.postingsFetchDuration = commonMetrics.fetchLatency.WithLabelValues(cacheTypePostings)
c.seriesFetchDuration = commonMetrics.fetchLatency.WithLabelValues(cacheTypeSeries)
c.expandedPostingsFetchDuration = commonMetrics.fetchLatency.WithLabelValues(cacheTypeExpandedPostings)

level.Info(logger).Log("msg", "created index cache")

return c, nil
Expand All @@ -88,6 +95,9 @@ func (c *RemoteIndexCache) StorePostings(blockID ulid.ULID, l labels.Label, v []
// and returns a map containing cache hits, along with a list of missing keys.
// In case of error, it logs and return an empty cache hits map.
func (c *RemoteIndexCache) FetchMultiPostings(ctx context.Context, blockID ulid.ULID, lbls []labels.Label) (hits map[labels.Label][]byte, misses []labels.Label) {
timer := prometheus.NewTimer(c.postingsFetchDuration)
defer timer.ObserveDuration()

keys := make([]string, 0, len(lbls))

blockIDKey := blockID.String()
Expand Down Expand Up @@ -138,6 +148,9 @@ func (c *RemoteIndexCache) StoreExpandedPostings(blockID ulid.ULID, keys []*labe
// and returns a map containing cache hits, along with a list of missing keys.
// In case of error, it logs and return an empty cache hits map.
func (c *RemoteIndexCache) FetchExpandedPostings(ctx context.Context, blockID ulid.ULID, lbls []*labels.Matcher) ([]byte, bool) {
timer := prometheus.NewTimer(c.postingsFetchDuration)
defer timer.ObserveDuration()

key := cacheKey{blockID.String(), cacheKeyExpandedPostings(labelMatchersToString(lbls)), c.compressionScheme}.string()

// Fetch the keys from memcached in a single request.
Expand Down Expand Up @@ -169,6 +182,9 @@ func (c *RemoteIndexCache) StoreSeries(blockID ulid.ULID, id storage.SeriesRef,
// and returns a map containing cache hits, along with a list of missing IDs.
// In case of error, it logs and return an empty cache hits map.
func (c *RemoteIndexCache) FetchMultiSeries(ctx context.Context, blockID ulid.ULID, ids []storage.SeriesRef) (hits map[storage.SeriesRef][]byte, misses []storage.SeriesRef) {
timer := prometheus.NewTimer(c.postingsFetchDuration)
defer timer.ObserveDuration()

keys := make([]string, 0, len(ids))

blockIDKey := blockID.String()
Expand Down
Loading