Skip to content

Commit

Permalink
add test case
Browse files Browse the repository at this point in the history
  • Loading branch information
@povilasv
povilasv committed Aug 19, 2019
1 parent 01389e8 commit 9a8d8ab
Showing 1 changed file with 126 additions and 32 deletions.
158 changes: 126 additions & 32 deletions pkg/store/bucket_e2e_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,14 @@ import (
"github.com/go-kit/kit/log"
"github.com/oklog/ulid"
"github.com/pkg/errors"
prommodel "github.com/prometheus/common/model"
"github.com/prometheus/prometheus/pkg/timestamp"
"github.com/prometheus/tsdb/labels"
"github.com/thanos-io/thanos/pkg/block"
"github.com/thanos-io/thanos/pkg/block/metadata"
"github.com/thanos-io/thanos/pkg/model"
"github.com/thanos-io/thanos/pkg/objstore"
"github.com/thanos-io/thanos/pkg/objstore/inmem"
"github.com/thanos-io/thanos/pkg/objstore/objtesting"
"github.com/thanos-io/thanos/pkg/runutil"
storecache "github.com/thanos-io/thanos/pkg/store/cache"
Expand Down Expand Up @@ -83,38 +85,20 @@ func (s *storeSuite) Close() {
s.wg.Wait()
}

func prepareStoreWithTestBlocks(t testing.TB, dir string, bkt objstore.Bucket, manyParts bool, maxSampleCount uint64) *storeSuite {
series := []labels.Labels{
labels.FromStrings("a", "1", "b", "1"),
labels.FromStrings("a", "1", "b", "2"),
labels.FromStrings("a", "2", "b", "1"),
labels.FromStrings("a", "2", "b", "2"),
labels.FromStrings("a", "1", "c", "1"),
labels.FromStrings("a", "1", "c", "2"),
labels.FromStrings("a", "2", "c", "1"),
labels.FromStrings("a", "2", "c", "2"),
}
extLset := labels.FromStrings("ext1", "value1")

start := time.Now()
now := start
func prepareTestBlocks(t testing.TB, now time.Time, count int, dir string, bkt objstore.Bucket,
series []labels.Labels, extLset labels.Labels) (blocks int, minTime, maxTime int64) {
ctx := context.Background()
logger := log.NewNopLogger()

ctx, cancel := context.WithCancel(context.Background())
s := &storeSuite{
cancel: cancel,
logger: log.NewLogfmtLogger(os.Stderr),
cache: &swappableCache{},
}
blocks := 0
for i := 0; i < 3; i++ {
for i := 0; i < count; i++ {
mint := timestamp.FromTime(now)
now = now.Add(2 * time.Hour)
maxt := timestamp.FromTime(now)

if s.minTime == 0 {
s.minTime = mint
if minTime == 0 {
minTime = mint
}
s.maxTime = maxt
maxTime = maxt

// Create two blocks per time slot. Only add 10 samples each so only one chunk
// gets created each. This way we can easily verify we got 10 chunks per series below.
Expand All @@ -129,19 +113,46 @@ func prepareStoreWithTestBlocks(t testing.TB, dir string, bkt objstore.Bucket, m
meta, err := metadata.Read(dir2)
testutil.Ok(t, err)
meta.Thanos.Labels = map[string]string{"ext2": "value2"}
testutil.Ok(t, metadata.Write(s.logger, dir2, meta))
testutil.Ok(t, metadata.Write(logger, dir2, meta))

testutil.Ok(t, block.Upload(ctx, s.logger, bkt, dir1))
testutil.Ok(t, block.Upload(ctx, s.logger, bkt, dir2))
testutil.Ok(t, block.Upload(ctx, logger, bkt, dir1))
testutil.Ok(t, block.Upload(ctx, logger, bkt, dir2))
blocks += 2

testutil.Ok(t, os.RemoveAll(dir1))
testutil.Ok(t, os.RemoveAll(dir2))
}

return
}

func prepareStoreWithTestBlocks(t testing.TB, dir string, bkt objstore.Bucket, manyParts bool, maxSampleCount uint64) *storeSuite {
series := []labels.Labels{
labels.FromStrings("a", "1", "b", "1"),
labels.FromStrings("a", "1", "b", "2"),
labels.FromStrings("a", "2", "b", "1"),
labels.FromStrings("a", "2", "b", "2"),
labels.FromStrings("a", "1", "c", "1"),
labels.FromStrings("a", "1", "c", "2"),
labels.FromStrings("a", "2", "c", "1"),
labels.FromStrings("a", "2", "c", "2"),
}
extLset := labels.FromStrings("ext1", "value1")

blocks, minTime, maxTime := prepareTestBlocks(t, time.Now(), 3, dir, bkt,
series, extLset)

ctx, cancel := context.WithCancel(context.Background())
s := &storeSuite{
cancel: cancel,
logger: log.NewLogfmtLogger(os.Stderr),
cache: &swappableCache{},
minTime: minTime,
maxTime: maxTime,
}

store, err := NewBucketStore(s.logger, nil, bkt, dir, s.cache, 0, maxSampleCount, 20, false, 20, filterConf)
testutil.Ok(t, err)

s.store = store

if manyParts {
Expand All @@ -155,8 +166,7 @@ func prepareStoreWithTestBlocks(t testing.TB, dir string, bkt objstore.Bucket, m
if err := runutil.Repeat(100*time.Millisecond, ctx.Done(), func() error {
return store.SyncBlocks(ctx)
}); err != nil && errors.Cause(err) != context.Canceled {
t.Error(err)
t.FailNow()
t.Fatal(err)
}
}()

Expand Down Expand Up @@ -444,3 +454,87 @@ func TestBucketStore_ManyParts_e2e(t *testing.T) {
testBucketStore_e2e(t, ctx, s)
})
}

func TestBucketStore_TimePartitioning_e2e(t *testing.T) {
bkt := inmem.NewBucket()

ctx, cancel := context.WithCancel(context.Background())
defer cancel()

dir, err := ioutil.TempDir("", "test_bucket_time_part_e2e")
testutil.Ok(t, err)
defer func() { testutil.Ok(t, os.RemoveAll(dir)) }()

series := []labels.Labels{
labels.FromStrings("a", "1", "b", "1"),
labels.FromStrings("a", "1", "b", "2"),
labels.FromStrings("a", "2", "b", "1"),
labels.FromStrings("a", "2", "b", "2"),
labels.FromStrings("a", "1", "c", "1"),
labels.FromStrings("a", "1", "c", "2"),
labels.FromStrings("a", "2", "c", "1"),
labels.FromStrings("a", "2", "c", "2"),
}
extLset := labels.FromStrings("ext1", "value1")

_, minTime, maxTime := prepareTestBlocks(t, time.Now(), 3, dir, bkt, series, extLset)

hourAfterDur := prommodel.Duration(1 * time.Hour)
hourAfter := model.TimeOrDurationValue{Dur: &hourAfterDur}

store, err := NewBucketStore(nil, nil, inmem.NewBucket(), dir, noopCache{}, 0, 0, 20, false, 20,
&FilterConfig{
MinTime: minTimeDuration,
MaxTime: hourAfter,
})
testutil.Ok(t, err)

err = store.InitialSync(ctx)
testutil.Ok(t, err)

mint, maxt := store.TimeRange()
testutil.Equals(t, minTime, mint)
testutil.Equals(t, maxTime, maxt)

vals, err := store.LabelValues(ctx, &storepb.LabelValuesRequest{Label: "a"})
testutil.Ok(t, err)
testutil.Equals(t, []string{"1", "2"}, vals.Values)

for i, tcase := range []struct {
req *storepb.SeriesRequest
expected [][]storepb.Label
}{
{
req: &storepb.SeriesRequest{
Matchers: []storepb.LabelMatcher{
{Type: storepb.LabelMatcher_RE, Name: "a", Value: "1|2"},
},
MinTime: mint,
MaxTime: maxt,
},
expected: [][]storepb.Label{
{{Name: "a", Value: "1"}, {Name: "b", Value: "1"}, {Name: "ext1", Value: "value1"}},
{{Name: "a", Value: "1"}, {Name: "b", Value: "2"}, {Name: "ext1", Value: "value1"}},
{{Name: "a", Value: "1"}, {Name: "c", Value: "1"}, {Name: "ext2", Value: "value2"}},
{{Name: "a", Value: "1"}, {Name: "c", Value: "2"}, {Name: "ext2", Value: "value2"}},
{{Name: "a", Value: "2"}, {Name: "b", Value: "1"}, {Name: "ext1", Value: "value1"}},
{{Name: "a", Value: "2"}, {Name: "b", Value: "2"}, {Name: "ext1", Value: "value1"}},
{{Name: "a", Value: "2"}, {Name: "c", Value: "1"}, {Name: "ext2", Value: "value2"}},
{{Name: "a", Value: "2"}, {Name: "c", Value: "2"}, {Name: "ext2", Value: "value2"}},
},
},
} {
t.Log("Run ", i)

srv := newStoreSeriesServer(ctx)

testutil.Ok(t, store.Series(tcase.req, srv))
testutil.Equals(t, len(tcase.expected), len(srv.SeriesSet))

for i, s := range srv.SeriesSet {
testutil.Equals(t, tcase.expected[i], s.Labels)
testutil.Equals(t, 3, len(s.Chunks))
}

}
}

0 comments on commit 9a8d8ab

Please sign in to comment.