Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

store: add acceptance tests for label methods to bucket store #6668

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions pkg/store/acceptance_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ type labelValuesCallCase struct {
}

// testLabelAPIs tests labels methods from StoreAPI from closed box perspective.
func testLabelAPIs(t *testing.T, startStore func(extLset labels.Labels, append func(app storage.Appender)) storepb.StoreServer) {
func testLabelAPIs(t *testing.T, startStore func(t *testing.T, extLset labels.Labels, append func(app storage.Appender)) storepb.StoreServer) {
t.Helper()

now := time.Now()
Expand Down Expand Up @@ -193,7 +193,8 @@ func testLabelAPIs(t *testing.T, startStore func(extLset labels.Labels, append f
if appendFn == nil {
appendFn = func(storage.Appender) {}
}
store := startStore(extLset, appendFn)
store := startStore(t, extLset, appendFn)

for _, c := range tc.labelNameCalls {
t.Run("label_names", func(t *testing.T) {
resp, err := store.LabelNames(context.Background(), &storepb.LabelNamesRequest{
Expand Down
78 changes: 77 additions & 1 deletion pkg/store/bucket_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1141,7 +1141,6 @@ func appendTestData(t testing.TB, app storage.Appender, series int) {
func createBlockFromHead(t testing.TB, dir string, head *tsdb.Head) ulid.ULID {
compactor, err := tsdb.NewLeveledCompactor(context.Background(), nil, log.NewNopLogger(), []int64{1000000}, nil, nil)
testutil.Ok(t, err)

testutil.Ok(t, os.MkdirAll(dir, 0777))

// Add +1 millisecond to block maxt because block intervals are half-open: [b.MinTime, b.MaxTime).
Expand Down Expand Up @@ -3342,3 +3341,80 @@ func TestBucketIndexReader_decodeCachedPostingsErrors(t *testing.T) {
testutil.NotOk(t, err)
})
}

func TestBucketStore_LabelAPIs(t *testing.T) {
t.Cleanup(func() { custom.TolerantVerifyLeak(t) })

testLabelAPIs(t, func(tt *testing.T, extLset labels.Labels, appendFn func(app storage.Appender)) storepb.StoreServer {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

tt is the testing.T from the particular subtest run; we need it because we dont want to skip all tests and also because we want to close resources properly

tmpDir := tt.TempDir()
bktDir := filepath.Join(tmpDir, "bkt")
auxDir := filepath.Join(tmpDir, "aux")
metaDir := filepath.Join(tmpDir, "meta")

testutil.Ok(tt, os.MkdirAll(metaDir, os.ModePerm))
testutil.Ok(tt, os.MkdirAll(auxDir, os.ModePerm))

bkt, err := filesystem.NewBucket(bktDir)
testutil.Ok(tt, err)
tt.Cleanup(func() { testutil.Ok(tt, bkt.Close()) })

headOpts := tsdb.DefaultHeadOptions()
headOpts.ChunkDirRoot = tmpDir
headOpts.ChunkRange = 1000
h, err := tsdb.NewHead(nil, nil, nil, nil, headOpts, nil)
testutil.Ok(tt, err)
tt.Cleanup(func() { testutil.Ok(tt, h.Close()) })
logger := log.NewNopLogger()

appendFn(h.Appender(context.Background()))

if h.NumSeries() == 0 {
tt.Skip("Bucket Store cannot handle empty HEAD")
}

id := createBlockFromHead(tt, auxDir, h)

auxBlockDir := filepath.Join(auxDir, id.String())
_, err = metadata.InjectThanos(log.NewNopLogger(), auxBlockDir, metadata.Thanos{
Labels: extLset.Map(),
Downsample: metadata.ThanosDownsample{Resolution: 0},
Source: metadata.TestSource,
}, nil)
testutil.Ok(tt, err)

testutil.Ok(tt, block.Upload(context.Background(), logger, bkt, auxBlockDir, metadata.NoneFunc))
testutil.Ok(tt, block.Upload(context.Background(), logger, bkt, auxBlockDir, metadata.NoneFunc))

chunkPool, err := NewDefaultChunkBytesPool(2e5)
testutil.Ok(tt, err)

metaFetcher, err := block.NewMetaFetcher(logger, 20, objstore.WithNoopInstr(bkt), metaDir, nil, []block.MetadataFilter{
block.NewTimePartitionMetaFilter(allowAllFilterConf.MinTime, allowAllFilterConf.MaxTime),
})
testutil.Ok(tt, err)

bucketStore, err := NewBucketStore(
objstore.WithNoopInstr(bkt),
metaFetcher,
"",
NewChunksLimiterFactory(10e6),
NewSeriesLimiterFactory(10e6),
NewBytesLimiterFactory(10e6),
NewGapBasedPartitioner(PartitionerMaxGapSize),
20,
true,
DefaultPostingOffsetInMemorySampling,
false,
false,
1*time.Minute,
WithChunkPool(chunkPool),
WithFilterConfig(allowAllFilterConf),
)
testutil.Ok(tt, err)
tt.Cleanup(func() { testutil.Ok(tt, bucketStore.Close()) })

testutil.Ok(tt, bucketStore.SyncBlocks(context.Background()))

return bucketStore
})
}
14 changes: 7 additions & 7 deletions pkg/store/prometheus_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -390,26 +390,26 @@ func TestPrometheusStore_SeriesLabels_e2e(t *testing.T) {

func TestPrometheusStore_LabelAPIs(t *testing.T) {
t.Cleanup(func() { custom.TolerantVerifyLeak(t) })
testLabelAPIs(t, func(extLset labels.Labels, appendFn func(app storage.Appender)) storepb.StoreServer {
testLabelAPIs(t, func(tt *testing.T, extLset labels.Labels, appendFn func(app storage.Appender)) storepb.StoreServer {
p, err := e2eutil.NewPrometheus()
testutil.Ok(t, err)
t.Cleanup(func() { testutil.Ok(t, p.Stop()) })
testutil.Ok(tt, err)
tt.Cleanup(func() { testutil.Ok(tt, p.Stop()) })

appendFn(p.Appender())

testutil.Ok(t, p.Start())
testutil.Ok(tt, p.Start())
u, err := url.Parse(fmt.Sprintf("http://%s", p.Addr()))
testutil.Ok(t, err)
testutil.Ok(tt, err)

version, err := promclient.NewDefaultClient().BuildVersion(context.Background(), u)
testutil.Ok(t, err)
testutil.Ok(tt, err)

promStore, err := NewPrometheusStore(nil, nil, promclient.NewDefaultClient(), u, component.Sidecar,
func() labels.Labels { return extLset },
nil,
func() stringset.Set { return stringset.AllStrings() },
func() string { return version })
testutil.Ok(t, err)
testutil.Ok(tt, err)

return promStore
})
Expand Down
6 changes: 3 additions & 3 deletions pkg/store/tsdb_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -230,10 +230,10 @@ func TestTSDBStore_Series(t *testing.T) {

func TestTSDBStore_LabelAPIs(t *testing.T) {
t.Cleanup(func() { custom.TolerantVerifyLeak(t) })
testLabelAPIs(t, func(extLset labels.Labels, appendFn func(app storage.Appender)) storepb.StoreServer {
testLabelAPIs(t, func(tt *testing.T, extLset labels.Labels, appendFn func(app storage.Appender)) storepb.StoreServer {
db, err := e2eutil.NewTSDB()
testutil.Ok(t, err)
t.Cleanup(func() { testutil.Ok(t, db.Close()) })
testutil.Ok(tt, err)
tt.Cleanup(func() { testutil.Ok(tt, db.Close()) })

tsdbStore := NewTSDBStore(nil, db, component.Rule, extLset)

Expand Down
Loading