feat(v2): metastore index (grafana#3586)

* Move error checking elsewhere

* Add metastore index (wip)

* Fix a few bugs and tests

* Remove unneeded file

* Fix race condition

* Enable new write path for local deployments

* Return level 0 blocks, remove duplicates

* Add simple logging

* Remove unused var

* Refactor out metastore index, add partition meta

* Refactor, minor perf improvements

* Fix local setup for v2

* Replace block truncation with block retention

* Refactor some more

* Add tests, basic config

* Add documentation

* Add todo

* Remove unused struct field tags

* Add todo

* Run make fmt

* Run make fmt

* Allow longer queries in v2 locally

* Add test for changing the partition duration

* Update test

* Address issue with finding / deleting blocks following a partition duration change

* Add test cases

* Remove nested locking primitives

* Remove index/models.go

* Improve error handling in ForEachPartition

* Fix access without a lock

* Validate block ids at the API level

* Improve FindBlocksInRange, add tests

* Switch to a LRU cache for loaded partitions

* Load partition meta on the fly

* Add logging

* Load the active partition in memory on startup

* Switch away from using a LRU cache for loaded partitions

* Switch the index to a partition+tenant structure

* Fix nil pointer dereference error on startup

* Fix compile error

* Update cache entry access ts on reads

* Add missing tenant check

* Run make fmt

* Fix panic in query frontend

* Fixed issue with unreachable anonymous blocks

.mockery.yaml

@@ -27,3 +27,7 @@ packages:
   github.com/grafana/pyroscope/pkg/experiment/metastore/dlq:
     interfaces:
       LocalServer:
+  github.com/grafana/pyroscope/pkg/experiment/metastore/index:
+    interfaces:
+      Store:
+        config:

pkg/experiment/ingester/segment_test.go

@@ -4,9 +4,6 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-	"github.com/grafana/dskit/flagext"
-	"github.com/grafana/pyroscope/pkg/experiment/metastore"
-	"github.com/grafana/pyroscope/pkg/test/mocks/mockdlq"
 	"io"
 	"math/rand"
 	"path/filepath"
@@ -16,7 +13,13 @@ import (
 	"testing"
 	"time"
 
+	"github.com/grafana/dskit/flagext"
+
+	"github.com/grafana/pyroscope/pkg/experiment/metastore"
+	"github.com/grafana/pyroscope/pkg/test/mocks/mockdlq"
+
 	gprofile "github.com/google/pprof/profile"
+
 	profilev1 "github.com/grafana/pyroscope/api/gen/proto/go/google/v1"
 	ingesterv1 "github.com/grafana/pyroscope/api/gen/proto/go/ingester/v1"
 	"github.com/grafana/pyroscope/api/gen/proto/go/ingester/v1/ingesterv1connect"
@@ -414,9 +417,9 @@ func TestDLQRecoveryMock(t *testing.T) {
 
 func TestDLQRecovery(t *testing.T) {
 	const tenant = "tb"
-	const ts = 239
+	var ts = time.Now().UnixMilli()
 	chunk := inputChunk([]input{
-		{shard: 1, tenant: tenant, profile: cpuProfile(42, ts, "svc1", "kek", "foo", "bar")},
+		{shard: 1, tenant: tenant, profile: cpuProfile(42, int(ts), "svc1", "kek", "foo", "bar")},
 	})
 
 	sw := newTestSegmentWriter(t, segmentWriterConfig{