fix(blooms): Use correct key to populate blockscache at startup

The cache key for block directories in the blockscache are the block's
address without the directory prefix. This is how the directory is put
to the LRU cache after downloading and extracing the block tarball.

This PR fixes the incorrect cache key used to populate the cache from
disk on startup, which contained the filesystem directory prefix.

Since the cached item from startup is never used, it gets evicted first
in case of a full cache, or due to its TTL. This causes also the
underlying directory on disk to be deleted, which can however still be
referenced from the correct cache key for that directory from a later
download.
That caused the error "getting index reader: opening series file: open
/path/to/block/series: no such file or directory" when trying to query
the block, because the correct cache key was still present.

Signed-off-by: Christian Haudum <christian.haudum@gmail.com>

pkg/storage/stores/shipper/bloomshipper/cache.go

@@ -100,9 +100,12 @@ func loadBlockDirectories(root string, logger log.Logger) (keys []string, values
 		}
 
 		if ok, clean := isBlockDir(path, logger); ok {
-			keys = append(keys, resolver.Block(ref).Addr())
+			// the cache key must not contain the directory prefix
+			// therefore we use the defaultKeyResolver to resolve the block's address
+			key := defaultKeyResolver{}.Block(ref).Addr()
+			keys = append(keys, key)
 			values = append(values, NewBlockDirectory(ref, path))
-			level.Debug(logger).Log("msg", "found block directory", "ref", ref, "path", path)
+			level.Debug(logger).Log("msg", "found block directory", "path", path, "key", key)
 		} else {
 			level.Warn(logger).Log("msg", "skip directory entry", "err", "not a block directory containing blooms and series", "path", path)
 			_ = clean(path)