store-gateway: remove fine-grained chunks caching

CHANGELOG.md

@@ -21,6 +21,7 @@
   * `-blocks-storage.bucket-store.index-header-lazy-loading-enabled` is deprecated, use the new configuration `-blocks-storage.bucket-store.index-header.lazy-loading-enabled`
   * `-blocks-storage.bucket-store.index-header-lazy-loading-idle-timeout` is deprecated, use the new configuration `-blocks-storage.bucket-store.index-header.lazy-loading-idle-timeout`
   * `-blocks-storage.bucket-store.index-header-lazy-loading-concurrency` is deprecated, use the new configuration `-blocks-storage.bucket-store.index-header.lazy-loading-concurrency`
+* [CHANGE] Store-gateway: remove experimental fine-grained chunks caching. The following experimental configuration parameters have been removed `-blocks-storage.bucket-store.chunks-cache.fine-grained-chunks-caching-enabled`, `-blocks-storage.bucket-store.fine-grained-chunks-caching-ranges-per-series`. #5816
 * [FEATURE] Introduced `distributor.service_overload_status_code_on_rate_limit_enabled` flag for configuring status code to 529 instead of 429 upon rate limit exhaustion. #5752
 * [FEATURE] Cardinality API: Add a new `count_method` parameter which enables counting active series #5136
 * [FEATURE] Query-frontend: added experimental support to cache cardinality, label names and label values query responses. The cache will be used when `-query-frontend.cache-results` is enabled, and `-query-frontend.results-cache-ttl-for-cardinality-query` or `-query-frontend.results-cache-ttl-for-labels-query` set to a value greater than 0. The following metrics have been added to track the query results cache hit ratio per `request_type`: #5212 #5235 #5426 #5524

cmd/mimir/config-descriptor.json

@@ -6840,17 +6840,6 @@
                   "fieldFlag": "blocks-storage.bucket-store.chunks-cache.subrange-ttl",
                   "fieldType": "duration",
                   "fieldCategory": "advanced"
-                },
-                {
-                  "kind": "field",
-                  "name": "fine_grained_chunks_caching_enabled",
-                  "required": false,
-                  "desc": "Enable fine-grained caching of chunks in the store-gateway. This reduces the required bandwidth and memory utilization.",
-                  "fieldValue": null,
-                  "fieldDefaultValue": false,
-                  "fieldFlag": "blocks-storage.bucket-store.chunks-cache.fine-grained-chunks-caching-enabled",
-                  "fieldType": "boolean",
-                  "fieldCategory": "experimental"
                 }
               ],
               "fieldValue": null,
@@ -7756,17 +7745,6 @@
               "fieldType": "int",
               "fieldCategory": "advanced"
             },
-            {
-              "kind": "field",
-              "name": "fine_grained_chunks_caching_ranges_per_series",
-              "required": false,
-              "desc": "This option controls into how many ranges the chunks of each series from each block are split. This value is effectively the number of chunks cache items per series per block when -blocks-storage.bucket-store.chunks-cache.fine-grained-chunks-caching-enabled is enabled.",
-              "fieldValue": null,
-              "fieldDefaultValue": 1,
-              "fieldFlag": "blocks-storage.bucket-store.fine-grained-chunks-caching-ranges-per-series",
-              "fieldType": "int",
-              "fieldCategory": "experimental"
-            },
             {
               "kind": "field",
               "name": "series_selection_strategy",

cmd/mimir/help-all.txt.tmpl

@@ -307,8 +307,6 @@ Usage of ./cmd/mimir/mimir:
     	TTL for caching object attributes for chunks. If the metadata cache is configured, attributes will be stored under this cache backend, otherwise attributes are stored in the chunks cache backend. (default 168h0m0s)
   -blocks-storage.bucket-store.chunks-cache.backend string
     	Backend for chunks cache, if not empty. Supported values: memcached, redis.
-  -blocks-storage.bucket-store.chunks-cache.fine-grained-chunks-caching-enabled
-    	[experimental] Enable fine-grained caching of chunks in the store-gateway. This reduces the required bandwidth and memory utilization.
   -blocks-storage.bucket-store.chunks-cache.max-get-range-requests int
     	Maximum number of sub-GetRange requests that a single GetRange request can be split into when fetching chunks. Zero or negative value = unlimited number of sub-requests. (default 3)
   -blocks-storage.bucket-store.chunks-cache.memcached.addresses comma-separated-list-of-strings
@@ -401,8 +399,6 @@ Usage of ./cmd/mimir/mimir:
     	Client write timeout. (default 3s)
   -blocks-storage.bucket-store.chunks-cache.subrange-ttl duration
     	TTL for caching individual chunks subranges. (default 24h0m0s)
-  -blocks-storage.bucket-store.fine-grained-chunks-caching-ranges-per-series int
-    	[experimental] This option controls into how many ranges the chunks of each series from each block are split. This value is effectively the number of chunks cache items per series per block when -blocks-storage.bucket-store.chunks-cache.fine-grained-chunks-caching-enabled is enabled. (default 1)
   -blocks-storage.bucket-store.ignore-blocks-within duration
     	Blocks with minimum time within this duration are ignored, and not loaded by store-gateway. Useful when used together with -querier.query-store-after to prevent loading young blocks, because there are usually many of them (depending on number of ingesters) and they are not yet compacted. Negative values or 0 disable the filter. (default 10h0m0s)
   -blocks-storage.bucket-store.ignore-deletion-marks-delay duration

docs/sources/mimir/configure/about-versioning.md

@@ -125,8 +125,6 @@ The following features are currently experimental:
 - Query-scheduler
   - `-query-scheduler.querier-forget-delay`
 - Store-gateway
-  - `-blocks-storage.bucket-store.chunks-cache.fine-grained-chunks-caching-enabled`
-  - `-blocks-storage.bucket-store.fine-grained-chunks-caching-ranges-per-series`
   - Use of Redis cache backend (`-blocks-storage.bucket-store.chunks-cache.backend=redis`, `-blocks-storage.bucket-store.index-cache.backend=redis`, `-blocks-storage.bucket-store.metadata-cache.backend=redis`)
   - `-blocks-storage.bucket-store.series-selection-strategy`
 - Read-write deployment mode

docs/sources/mimir/references/configuration-parameters/index.md

@@ -3300,11 +3300,6 @@ bucket_store:
     # CLI flag: -blocks-storage.bucket-store.chunks-cache.subrange-ttl
     [subrange_ttl: <duration> | default = 24h]
 
-    # (experimental) Enable fine-grained caching of chunks in the store-gateway.
-    # This reduces the required bandwidth and memory utilization.
-    # CLI flag: -blocks-storage.bucket-store.chunks-cache.fine-grained-chunks-caching-enabled
-    [fine_grained_chunks_caching_enabled: <boolean> | default = false]
-
   metadata_cache:
     # Backend for metadata cache, if not empty. Supported values: memcached,
     # redis.
@@ -3502,14 +3497,6 @@ bucket_store:
   # CLI flag: -blocks-storage.bucket-store.batch-series-size
   [streaming_series_batch_size: <int> | default = 5000]
 
-  # (experimental) This option controls into how many ranges the chunks of each
-  # series from each block are split. This value is effectively the number of
-  # chunks cache items per series per block when
-  # -blocks-storage.bucket-store.chunks-cache.fine-grained-chunks-caching-enabled
-  # is enabled.
-  # CLI flag: -blocks-storage.bucket-store.fine-grained-chunks-caching-ranges-per-series
-  [fine_grained_chunks_caching_ranges_per_series: <int> | default = 1]
-
   # (experimental) This option controls the strategy to selection of series and
   # deferring application of matchers. A more aggressive strategy will fetch
   # less posting lists at the cost of more series. This is useful when querying

pkg/storage/tsdb/caching_config.go

@@ -33,11 +33,10 @@ var supportedCacheBackends = []string{cache.BackendMemcached, cache.BackendRedis
 type ChunksCacheConfig struct {
 	cache.BackendConfig `yaml:",inline"`
 
-	MaxGetRangeRequests             int           `yaml:"max_get_range_requests" category:"advanced"`
-	AttributesTTL                   time.Duration `yaml:"attributes_ttl" category:"advanced"`
-	AttributesInMemoryMaxItems      int           `yaml:"attributes_in_memory_max_items" category:"advanced"`
-	SubrangeTTL                     time.Duration `yaml:"subrange_ttl" category:"advanced"`
-	FineGrainedChunksCachingEnabled bool          `yaml:"fine_grained_chunks_caching_enabled" category:"experimental"`
+	MaxGetRangeRequests        int           `yaml:"max_get_range_requests" category:"advanced"`
+	AttributesTTL              time.Duration `yaml:"attributes_ttl" category:"advanced"`
+	AttributesInMemoryMaxItems int           `yaml:"attributes_in_memory_max_items" category:"advanced"`
+	SubrangeTTL                time.Duration `yaml:"subrange_ttl" category:"advanced"`
 }
 
 func (cfg *ChunksCacheConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string) {
@@ -50,7 +49,6 @@ func (cfg *ChunksCacheConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix st
 	f.DurationVar(&cfg.AttributesTTL, prefix+"attributes-ttl", 168*time.Hour, "TTL for caching object attributes for chunks. If the metadata cache is configured, attributes will be stored under this cache backend, otherwise attributes are stored in the chunks cache backend.")
 	f.IntVar(&cfg.AttributesInMemoryMaxItems, prefix+"attributes-in-memory-max-items", 50000, "Maximum number of object attribute items to keep in a first level in-memory LRU cache. Metadata will be stored and fetched in-memory before hitting the cache backend. 0 to disable the in-memory cache.")
 	f.DurationVar(&cfg.SubrangeTTL, prefix+"subrange-ttl", 24*time.Hour, "TTL for caching individual chunks subranges.")
-	f.BoolVar(&cfg.FineGrainedChunksCachingEnabled, prefix+"fine-grained-chunks-caching-enabled", false, "Enable fine-grained caching of chunks in the store-gateway. This reduces the required bandwidth and memory utilization.")
 }
 
 func (cfg *ChunksCacheConfig) Validate() error {
@@ -138,9 +136,7 @@ func CreateCachingBucket(chunksCache cache.Cache, chunksConfig ChunksCacheConfig
 				return nil, errors.Wrapf(err, "wrap metadata cache with in-memory cache")
 			}
 		}
-		if !chunksConfig.FineGrainedChunksCachingEnabled {
-			cfg.CacheGetRange("chunks", chunksCache, isTSDBChunkFile, subrangeSize, attributesCache, chunksConfig.AttributesTTL, chunksConfig.SubrangeTTL, chunksConfig.MaxGetRangeRequests)
-		}
+		cfg.CacheGetRange("chunks", chunksCache, isTSDBChunkFile, subrangeSize, attributesCache, chunksConfig.AttributesTTL, chunksConfig.SubrangeTTL, chunksConfig.MaxGetRangeRequests)
 	}
 
 	if !cachingConfigured {

pkg/storage/tsdb/config.go

@@ -404,7 +404,6 @@ type BucketStoreConfig struct {
 	IndexHeader indexheader.Config `yaml:"index_header" category:"experimental"`
 
 	StreamingBatchSize          int    `yaml:"streaming_series_batch_size" category:"advanced"`
-	ChunkRangesPerSeries        int    `yaml:"fine_grained_chunks_caching_ranges_per_series" category:"experimental"`
 	SeriesSelectionStrategyName string `yaml:"series_selection_strategy" category:"experimental"`
 	SelectionStrategies         struct {
 		WorstCaseSeriesPreference float64 `yaml:"worst_case_series_preference" category:"experimental"`
@@ -451,7 +450,6 @@ func (cfg *BucketStoreConfig) RegisterFlags(f *flag.FlagSet) {
 	f.DurationVar(&cfg.DeprecatedIndexHeaderLazyLoadingIdleTimeout, "blocks-storage.bucket-store.index-header-lazy-loading-idle-timeout", indexheader.DefaultIndexHeaderLazyLoadingIdleTimeout, "If index-header lazy loading is enabled and this setting is > 0, the store-gateway will offload unused index-headers after 'idle timeout' inactivity.")
 	f.Uint64Var(&cfg.PartitionerMaxGapBytes, "blocks-storage.bucket-store.partitioner-max-gap-bytes", DefaultPartitionerMaxGapSize, "Max size - in bytes - of a gap for which the partitioner aggregates together two bucket GET object requests.")
 	f.IntVar(&cfg.StreamingBatchSize, "blocks-storage.bucket-store.batch-series-size", 5000, "This option controls how many series to fetch per batch. The batch size must be greater than 0.")
-	f.IntVar(&cfg.ChunkRangesPerSeries, "blocks-storage.bucket-store.fine-grained-chunks-caching-ranges-per-series", 1, "This option controls into how many ranges the chunks of each series from each block are split. This value is effectively the number of chunks cache items per series per block when -blocks-storage.bucket-store.chunks-cache.fine-grained-chunks-caching-enabled is enabled.")
 	f.StringVar(&cfg.SeriesSelectionStrategyName, seriesSelectionStrategyFlag, WorstCasePostingsStrategy, "This option controls the strategy to selection of series and deferring application of matchers. A more aggressive strategy will fetch less posting lists at the cost of more series. This is useful when querying large blocks in which many series share the same label name and value. Supported values (most aggressive to least aggressive): "+strings.Join(validSeriesSelectionStrategies, ", ")+".")
 	f.Float64Var(&cfg.SelectionStrategies.WorstCaseSeriesPreference, "blocks-storage.bucket-store.series-selection-strategies.worst-case-series-preference", 0.75, "This option is only used when "+seriesSelectionStrategyFlag+"="+WorstCasePostingsStrategy+". Increasing the series preference results in fetching more series than postings. Must be a positive floating point number.")
 }

pkg/storegateway/bucket.go

@@ -45,7 +45,6 @@ import (
 	"github.com/grafana/mimir/pkg/storage/tsdb"
 	"github.com/grafana/mimir/pkg/storage/tsdb/block"
 	"github.com/grafana/mimir/pkg/storage/tsdb/bucketcache"
-	"github.com/grafana/mimir/pkg/storegateway/chunkscache"
 	"github.com/grafana/mimir/pkg/storegateway/hintspb"
 	"github.com/grafana/mimir/pkg/storegateway/indexcache"
 	"github.com/grafana/mimir/pkg/storegateway/indexheader"
@@ -92,7 +91,6 @@ type BucketStore struct {
 	fetcher         block.MetadataFetcher
 	dir             string
 	indexCache      indexcache.IndexCache
-	chunksCache     chunkscache.Cache
 	indexReaderPool *indexheader.ReaderPool
 	seriesHashCache *hashcache.SeriesHashCache
 
@@ -109,14 +107,6 @@ type BucketStore struct {
 	// This value must be greater than zero.
 	maxSeriesPerBatch int
 
-	// numChunksRangesPerSeries controls into how many ranges the chunks of each series from each block are split.
-	// This value is effectively the number of chunks cache items per series per block.
-	numChunksRangesPerSeries int
-
-	// fineGrainedChunksCachingEnabled controls whether to use the per series chunks caching
-	// or rely on the transparent caching bucket.
-	fineGrainedChunksCachingEnabled bool
-
 	// Query gate which limits the maximum amount of concurrent queries.
 	queryGate gate.Gate
 
@@ -194,13 +184,6 @@ func WithIndexCache(cache indexcache.IndexCache) BucketStoreOption {
 	}
 }
 
-// WithChunksCache sets a chunksCache to use instead of a noopCache.
-func WithChunksCache(cache chunkscache.Cache) BucketStoreOption {
-	return func(s *BucketStore) {
-		s.chunksCache = cache
-	}
-}
-
 // WithQueryGate sets a queryGate to use instead of a gate.NewNoop().
 func WithQueryGate(queryGate gate.Gate) BucketStoreOption {
 	return func(s *BucketStore) {
@@ -215,12 +198,6 @@ func WithLazyLoadingGate(lazyLoadingGate gate.Gate) BucketStoreOption {
 	}
 }
 
-func WithFineGrainedChunksCaching(enabled bool) BucketStoreOption {
-	return func(s *BucketStore) {
-		s.fineGrainedChunksCachingEnabled = enabled
-	}
-}
-
 // NewBucketStore creates a new bucket backed store that implements the store API against
 // an object store bucket. It is optimized to work against high latency backends.
 func NewBucketStore(
@@ -243,7 +220,6 @@ func NewBucketStore(
 		fetcher:                     fetcher,
 		dir:                         dir,
 		indexCache:                  noopCache{},
-		chunksCache:                 chunkscache.NoopCache{},
 		blocks:                      map[ulid.ULID]*bucketBlock{},
 		blockSet:                    newBucketBlockSet(),
 		blockSyncConcurrency:        bucketStoreConfig.BlockSyncConcurrency,
@@ -258,7 +234,6 @@ func NewBucketStore(
 		metrics:                     metrics,
 		userID:                      userID,
 		maxSeriesPerBatch:           bucketStoreConfig.StreamingBatchSize,
-		numChunksRangesPerSeries:    bucketStoreConfig.ChunkRangesPerSeries,
 		postingsStrategy:            postingsStrategy,
 	}
 
@@ -1032,11 +1007,7 @@ func (s *BucketStore) nonStreamingSeriesSetForBlocks(
 
 	var set storepb.SeriesSet
 	if !req.SkipChunks {
-		var cache chunkscache.Cache
-		if s.fineGrainedChunksCachingEnabled {
-			cache = s.chunksCache
-		}
-		ss := newChunksPreloadingIterator(ctx, s.logger, s.userID, cache, *chunkReaders, it, s.maxSeriesPerBatch, stats, req.MinTime, req.MaxTime)
+		ss := newChunksPreloadingIterator(ctx, s.logger, s.userID, *chunkReaders, it, s.maxSeriesPerBatch, stats, req.MinTime, req.MaxTime)
 		set = newSeriesChunksSeriesSet(ss)
 	} else {
 		set = newSeriesSetWithoutChunks(ctx, it, stats)
@@ -1092,12 +1063,7 @@ func (s *BucketStore) streamingChunksSetForBlocks(
 	if err != nil {
 		return nil, err
 	}
-
-	var cache chunkscache.Cache
-	if s.fineGrainedChunksCachingEnabled {
-		cache = s.chunksCache
-	}
-	scsi := newChunksPreloadingIterator(ctx, s.logger, s.userID, cache, *chunkReaders, it, s.maxSeriesPerBatch, stats, req.MinTime, req.MaxTime)
+	scsi := newChunksPreloadingIterator(ctx, s.logger, s.userID, *chunkReaders, it, s.maxSeriesPerBatch, stats, req.MinTime, req.MaxTime)
 	return scsi, nil
 }
 
@@ -1150,7 +1116,6 @@ func (s *BucketStore) getSeriesIteratorFromBlocks(
 				cachedSeriesHasher{blockSeriesHashCache},
 				strategy,
 				req.MinTime, req.MaxTime,
-				s.numChunksRangesPerSeries,
 				stats,
 				r,
 				s.logger,
@@ -1205,20 +1170,11 @@ func (s *BucketStore) recordSeriesCallResult(safeStats *safeQueryStats) {
 
 	s.metrics.seriesBlocksQueried.Observe(float64(stats.blocksQueried))
 
-	if s.fineGrainedChunksCachingEnabled {
-		s.metrics.seriesDataTouched.WithLabelValues("chunks", "processed").Observe(float64(stats.chunksProcessed))
-		s.metrics.seriesDataSizeTouched.WithLabelValues("chunks", "processed").Observe(float64(stats.chunksProcessedSizeSum))
-		// With fine-grained caching we may have touched more chunks than we need because we had to fetch a
-		// whole range of chunks, which includes chunks outside the request's minT/maxT.
-		s.metrics.seriesDataTouched.WithLabelValues("chunks", "returned").Observe(float64(stats.chunksReturned))
-		s.metrics.seriesDataSizeTouched.WithLabelValues("chunks", "returned").Observe(float64(stats.chunksReturnedSizeSum))
-	} else {
-		s.metrics.seriesDataTouched.WithLabelValues("chunks", "processed").Observe(float64(stats.chunksTouched))
-		s.metrics.seriesDataSizeTouched.WithLabelValues("chunks", "processed").Observe(float64(stats.chunksTouchedSizeSum))
-		// For the implementation which uses the caching bucket the bytes we touch are the bytes we return.
-		s.metrics.seriesDataTouched.WithLabelValues("chunks", "returned").Observe(float64(stats.chunksTouched))
-		s.metrics.seriesDataSizeTouched.WithLabelValues("chunks", "returned").Observe(float64(stats.chunksTouchedSizeSum))
-	}
+	s.metrics.seriesDataTouched.WithLabelValues("chunks", "processed").Observe(float64(stats.chunksTouched))
+	s.metrics.seriesDataSizeTouched.WithLabelValues("chunks", "processed").Observe(float64(stats.chunksTouchedSizeSum))
+	// For the implementation which uses the caching bucket the bytes we touch are the bytes we return.
+	s.metrics.seriesDataTouched.WithLabelValues("chunks", "returned").Observe(float64(stats.chunksTouched))
+	s.metrics.seriesDataSizeTouched.WithLabelValues("chunks", "returned").Observe(float64(stats.chunksTouchedSizeSum))
 
 	s.metrics.resultSeriesCount.Observe(float64(stats.mergedSeriesCount))
 }
@@ -1440,7 +1396,6 @@ func blockLabelNames(ctx context.Context, indexr *bucketIndexReader, matchers []
 		cachedSeriesHasher{nil},
 		noChunkRefs,
 		minTime, maxTime,
-		1, // we skip chunks, so this doesn't make any difference
 		stats,
 		nil,
 		logger,
@@ -1662,7 +1617,6 @@ func labelValuesFromSeries(ctx context.Context, labelName string, seriesPerBatch
 		b.meta.MinTime,
 		b.meta.MaxTime,
 		b.userID,
-		1,
 		b.logger,
 	)
 	if len(pendingMatchers) > 0 {

pkg/storegateway/bucket_chunk_reader_test.go

@@ -41,7 +41,6 @@ func TestBucketChunkReader_refetchChunks(t *testing.T) {
 		defaultStrategy,
 		block.meta.MinTime,
 		block.meta.MaxTime,
-		2,
 		newSafeQueryStats(),
 		nil,
 		log.NewNopLogger(),