testutil: Moving all e2e utils to separate package, added TB union.

Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
thanos-io · Jan 30, 2020 · e232db1 · e232db1
1 parent 72bac9e
commit e232db1
Show file tree

Hide file tree

Showing 22 changed files with 194 additions and 60 deletions.
diff --git a/cmd/thanos/main_test.go b/cmd/thanos/main_test.go
@@ -9,9 +9,6 @@ import (
 	"os"
 	"path"
 	"path/filepath"
-
-	"github.com/thanos-io/thanos/pkg/block/metadata"
-
 	"testing"
 	"time"
 
@@ -21,10 +18,12 @@ import (
 	promtest "github.com/prometheus/client_golang/prometheus/testutil"
 	"github.com/prometheus/prometheus/pkg/labels"
 	"github.com/thanos-io/thanos/pkg/block"
+	"github.com/thanos-io/thanos/pkg/block/metadata"
 	"github.com/thanos-io/thanos/pkg/compact"
 	"github.com/thanos-io/thanos/pkg/compact/downsample"
 	"github.com/thanos-io/thanos/pkg/objstore/inmem"
 	"github.com/thanos-io/thanos/pkg/testutil"
+	"github.com/thanos-io/thanos/pkg/testutil/e2eutil"
 )
 
 func TestCleanupIndexCacheFolder(t *testing.T) {
@@ -41,7 +40,7 @@ func TestCleanupIndexCacheFolder(t *testing.T) {
 	// Upload one compaction lvl = 2 block, one compaction lvl = 1.
 	// We generate index cache files only for lvl > 1 blocks.
 	{
-		id, err := testutil.CreateBlock(
+		id, err := e2eutil.CreateBlock(
 			ctx,
 			dir,
 			[]labels.Labels{{{Name: "a", Value: "1"}}},
@@ -59,7 +58,7 @@ func TestCleanupIndexCacheFolder(t *testing.T) {
 		testutil.Ok(t, block.Upload(ctx, logger, bkt, path.Join(dir, id.String())))
 	}
 	{
-		id, err := testutil.CreateBlock(
+		id, err := e2eutil.CreateBlock(
 			ctx,
 			dir,
 			[]labels.Labels{{{Name: "a", Value: "1"}}},
@@ -102,7 +101,7 @@ func TestCleanupDownsampleCacheFolder(t *testing.T) {
 	bkt := inmem.NewBucket()
 	var id ulid.ULID
 	{
-		id, err = testutil.CreateBlock(
+		id, err = e2eutil.CreateBlock(
 			ctx,
 			dir,
 			[]labels.Labels{{{Name: "a", Value: "1"}}},

diff --git a/pkg/block/block_test.go b/pkg/block/block_test.go
@@ -17,6 +17,7 @@ import (
 	"github.com/prometheus/prometheus/pkg/labels"
 	"github.com/thanos-io/thanos/pkg/objstore/inmem"
 	"github.com/thanos-io/thanos/pkg/testutil"
+	"github.com/thanos-io/thanos/pkg/testutil/e2eutil"
 
 	"github.com/oklog/ulid"
 )
@@ -77,7 +78,7 @@ func TestUpload(t *testing.T) {
 	defer func() { testutil.Ok(t, os.RemoveAll(tmpDir)) }()
 
 	bkt := inmem.NewBucket()
-	b1, err := testutil.CreateBlock(ctx, tmpDir, []labels.Labels{
+	b1, err := e2eutil.CreateBlock(ctx, tmpDir, []labels.Labels{
 		{{Name: "a", Value: "1"}},
 		{{Name: "a", Value: "2"}},
 		{{Name: "a", Value: "3"}},
@@ -105,7 +106,7 @@ func TestUpload(t *testing.T) {
 		testutil.NotOk(t, err)
 		testutil.Assert(t, strings.HasSuffix(err.Error(), "/meta.json: no such file or directory"), "")
 	}
-	testutil.Copy(t, path.Join(tmpDir, b1.String(), MetaFilename), path.Join(tmpDir, "test", b1.String(), MetaFilename))
+	e2eutil.Copy(t, path.Join(tmpDir, b1.String(), MetaFilename), path.Join(tmpDir, "test", b1.String(), MetaFilename))
 	{
 		// Missing chunks.
 		err := Upload(ctx, log.NewNopLogger(), bkt, path.Join(tmpDir, "test", b1.String()))
@@ -116,7 +117,7 @@ func TestUpload(t *testing.T) {
 		testutil.Equals(t, 1, len(bkt.Objects()))
 	}
 	testutil.Ok(t, os.MkdirAll(path.Join(tmpDir, "test", b1.String(), ChunksDirname), os.ModePerm))
-	testutil.Copy(t, path.Join(tmpDir, b1.String(), ChunksDirname, "000001"), path.Join(tmpDir, "test", b1.String(), ChunksDirname, "000001"))
+	e2eutil.Copy(t, path.Join(tmpDir, b1.String(), ChunksDirname, "000001"), path.Join(tmpDir, "test", b1.String(), ChunksDirname, "000001"))
 	{
 		// Missing index file.
 		err := Upload(ctx, log.NewNopLogger(), bkt, path.Join(tmpDir, "test", b1.String()))
@@ -126,7 +127,7 @@ func TestUpload(t *testing.T) {
 		// Only debug meta.json present.
 		testutil.Equals(t, 1, len(bkt.Objects()))
 	}
-	testutil.Copy(t, path.Join(tmpDir, b1.String(), IndexFilename), path.Join(tmpDir, "test", b1.String(), IndexFilename))
+	e2eutil.Copy(t, path.Join(tmpDir, b1.String(), IndexFilename), path.Join(tmpDir, "test", b1.String(), IndexFilename))
 	testutil.Ok(t, os.Remove(path.Join(tmpDir, "test", b1.String(), MetaFilename)))
 	{
 		// Missing meta.json file.
@@ -137,7 +138,7 @@ func TestUpload(t *testing.T) {
 		// Only debug meta.json present.
 		testutil.Equals(t, 1, len(bkt.Objects()))
 	}
-	testutil.Copy(t, path.Join(tmpDir, b1.String(), MetaFilename), path.Join(tmpDir, "test", b1.String(), MetaFilename))
+	e2eutil.Copy(t, path.Join(tmpDir, b1.String(), MetaFilename), path.Join(tmpDir, "test", b1.String(), MetaFilename))
 	{
 		// Full block.
 		testutil.Ok(t, Upload(ctx, log.NewNopLogger(), bkt, path.Join(tmpDir, "test", b1.String())))
@@ -156,7 +157,7 @@ func TestUpload(t *testing.T) {
 	}
 	{
 		// Upload with no external labels should be blocked.
-		b2, err := testutil.CreateBlock(ctx, tmpDir, []labels.Labels{
+		b2, err := e2eutil.CreateBlock(ctx, tmpDir, []labels.Labels{
 			{{Name: "a", Value: "1"}},
 			{{Name: "a", Value: "2"}},
 			{{Name: "a", Value: "3"}},
@@ -182,7 +183,7 @@ func TestDelete(t *testing.T) {
 
 	bkt := inmem.NewBucket()
 	{
-		b1, err := testutil.CreateBlock(ctx, tmpDir, []labels.Labels{
+		b1, err := e2eutil.CreateBlock(ctx, tmpDir, []labels.Labels{
 			{{Name: "a", Value: "1"}},
 			{{Name: "a", Value: "2"}},
 			{{Name: "a", Value: "3"}},
@@ -199,7 +200,7 @@ func TestDelete(t *testing.T) {
 		testutil.Equals(t, 1, len(bkt.Objects()))
 	}
 	{
-		b2, err := testutil.CreateBlock(ctx, tmpDir, []labels.Labels{
+		b2, err := e2eutil.CreateBlock(ctx, tmpDir, []labels.Labels{
 			{{Name: "a", Value: "1"}},
 			{{Name: "a", Value: "2"}},
 			{{Name: "a", Value: "3"}},

diff --git a/pkg/block/index_test.go b/pkg/block/index_test.go
@@ -17,6 +17,7 @@ import (
 	"github.com/prometheus/prometheus/tsdb/index"
 	"github.com/thanos-io/thanos/pkg/block/metadata"
 	"github.com/thanos-io/thanos/pkg/testutil"
+	"github.com/thanos-io/thanos/pkg/testutil/e2eutil"
 )
 
 func TestRewrite(t *testing.T) {
@@ -26,7 +27,7 @@ func TestRewrite(t *testing.T) {
 	testutil.Ok(t, err)
 	defer func() { testutil.Ok(t, os.RemoveAll(tmpDir)) }()
 
-	b, err := testutil.CreateBlock(ctx, tmpDir, []labels.Labels{
+	b, err := e2eutil.CreateBlock(ctx, tmpDir, []labels.Labels{
 		{{Name: "a", Value: "1"}},
 		{{Name: "a", Value: "2"}},
 		{{Name: "a", Value: "3"}},

diff --git a/pkg/block/indexheader/header_test.go b/pkg/block/indexheader/header_test.go
@@ -25,6 +25,7 @@ import (
 	"github.com/thanos-io/thanos/pkg/objstore/filesystem"
 	"github.com/thanos-io/thanos/pkg/runutil"
 	"github.com/thanos-io/thanos/pkg/testutil"
+	"github.com/thanos-io/thanos/pkg/testutil/e2eutil"
 )
 
 func TestReaders(t *testing.T) {
@@ -39,7 +40,7 @@ func TestReaders(t *testing.T) {
 	defer func() { testutil.Ok(t, bkt.Close()) }()
 
 	// Create block index version 2.
-	id1, err := testutil.CreateBlock(ctx, tmpDir, []labels.Labels{
+	id1, err := e2eutil.CreateBlock(ctx, tmpDir, []labels.Labels{
 		{{Name: "a", Value: "1"}},
 		{{Name: "a", Value: "2"}},
 		{{Name: "a", Value: "3"}},
@@ -69,7 +70,7 @@ func TestReaders(t *testing.T) {
 
 	m, err := metadata.Read("./testdata/index_format_v1")
 	testutil.Ok(t, err)
-	testutil.Copy(t, "./testdata/index_format_v1", filepath.Join(tmpDir, m.ULID.String()))
+	e2eutil.Copy(t, "./testdata/index_format_v1", filepath.Join(tmpDir, m.ULID.String()))
 
 	_, err = metadata.InjectThanos(log.NewNopLogger(), filepath.Join(tmpDir, m.ULID.String()), metadata.Thanos{
 		Labels:     labels.Labels{{Name: "ext1", Value: "1"}}.Map(),
@@ -264,7 +265,7 @@ func prepareIndexV2Block(t testing.TB, tmpDir string, bkt objstore.Bucket) *meta
 
 	m, err := metadata.Read("./testdata/index_format_v2")
 	testutil.Ok(t, err)
-	testutil.Copy(t, "./testdata/index_format_v2", filepath.Join(tmpDir, m.ULID.String()))
+	e2eutil.Copy(t, "./testdata/index_format_v2", filepath.Join(tmpDir, m.ULID.String()))
 
 	_, err = metadata.InjectThanos(log.NewNopLogger(), filepath.Join(tmpDir, m.ULID.String()), metadata.Thanos{
 		Labels:     labels.Labels{{Name: "ext1", Value: "1"}}.Map(),

diff --git a/pkg/compact/compact_e2e_test.go b/pkg/compact/compact_e2e_test.go
@@ -30,6 +30,7 @@ import (
 	"github.com/thanos-io/thanos/pkg/objstore"
 	"github.com/thanos-io/thanos/pkg/objstore/objtesting"
 	"github.com/thanos-io/thanos/pkg/testutil"
+	"github.com/thanos-io/thanos/pkg/testutil/e2eutil"
 )
 
 func TestSyncer_GarbageCollect_e2e(t *testing.T) {
@@ -383,7 +384,7 @@ func createAndUpload(t testing.TB, bkt objstore.Bucket, blocks []blockgenSpec) (
 		if b.numSamples == 0 {
 			id, err = createEmptyBlock(prepareDir, b.mint, b.maxt, b.extLset, b.res)
 		} else {
-			id, err = testutil.CreateBlock(ctx, prepareDir, b.series, b.numSamples, b.mint, b.maxt, b.extLset, b.res)
+			id, err = e2eutil.CreateBlock(ctx, prepareDir, b.series, b.numSamples, b.mint, b.maxt, b.extLset, b.res)
 		}
 		testutil.Ok(t, err)
 

diff --git a/pkg/promclient/promclient_e2e_test.go b/pkg/promclient/promclient_e2e_test.go
@@ -20,10 +20,11 @@ import (
 	"github.com/prometheus/prometheus/pkg/timestamp"
 	"github.com/thanos-io/thanos/pkg/runutil"
 	"github.com/thanos-io/thanos/pkg/testutil"
+	"github.com/thanos-io/thanos/pkg/testutil/e2eutil"
 )
 
 func TestIsWALFileAccessible_e2e(t *testing.T) {
-	testutil.ForeachPrometheus(t, func(t testing.TB, p *testutil.Prometheus) {
+	e2eutil.ForeachPrometheus(t, func(t testing.TB, p *e2eutil.Prometheus) {
 		testutil.Ok(t, p.Start())
 
 		ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
@@ -36,7 +37,7 @@ func TestIsWALFileAccessible_e2e(t *testing.T) {
 }
 
 func TestExternalLabels_e2e(t *testing.T) {
-	testutil.ForeachPrometheus(t, func(t testing.TB, p *testutil.Prometheus) {
+	e2eutil.ForeachPrometheus(t, func(t testing.TB, p *e2eutil.Prometheus) {
 		testutil.Ok(t, p.SetConfig(`
 global:
   external_labels:
@@ -59,7 +60,7 @@ global:
 }
 
 func TestConfiguredFlags_e2e(t *testing.T) {
-	testutil.ForeachPrometheus(t, func(t testing.TB, p *testutil.Prometheus) {
+	e2eutil.ForeachPrometheus(t, func(t testing.TB, p *e2eutil.Prometheus) {
 		testutil.Ok(t, p.Start())
 
 		u, err := url.Parse(fmt.Sprintf("http://%s", p.Addr()))
@@ -78,12 +79,12 @@ func TestConfiguredFlags_e2e(t *testing.T) {
 }
 
 func TestSnapshot_e2e(t *testing.T) {
-	testutil.ForeachPrometheus(t, func(t testing.TB, p *testutil.Prometheus) {
+	e2eutil.ForeachPrometheus(t, func(t testing.TB, p *e2eutil.Prometheus) {
 		now := time.Now()
 
 		ctx := context.Background()
 		// Create artificial block.
-		id, err := testutil.CreateBlockWithTombstone(
+		id, err := e2eutil.CreateBlockWithTombstone(
 			ctx,
 			p.Dir(),
 			[]labels.Labels{labels.FromStrings("a", "b")},

diff --git a/pkg/query/api/v1_test.go b/pkg/query/api/v1_test.go
@@ -45,6 +45,7 @@ import (
 	"github.com/thanos-io/thanos/pkg/query"
 	"github.com/thanos-io/thanos/pkg/store"
 	"github.com/thanos-io/thanos/pkg/testutil"
+	"github.com/thanos-io/thanos/pkg/testutil/e2eutil"
 )
 
 func TestEndpoints(t *testing.T) {
@@ -85,7 +86,7 @@ func TestEndpoints(t *testing.T) {
 		},
 	}
 
-	db, err := testutil.NewTSDB()
+	db, err := e2eutil.NewTSDB()
 	defer func() { testutil.Ok(t, db.Close()) }()
 	testutil.Ok(t, err)
 

diff --git a/pkg/shipper/shipper_e2e_test.go b/pkg/shipper/shipper_e2e_test.go
@@ -16,6 +16,7 @@ import (
 	"time"
 
 	"github.com/thanos-io/thanos/pkg/objstore/inmem"
+	"github.com/thanos-io/thanos/pkg/testutil/e2eutil"
 
 	"github.com/go-kit/kit/log"
 	"github.com/oklog/ulid"
@@ -172,7 +173,7 @@ func TestShipper_SyncBlocks_e2e(t *testing.T) {
 }
 
 func TestShipper_SyncBlocksWithMigrating_e2e(t *testing.T) {
-	testutil.ForeachPrometheus(t, func(t testing.TB, p *testutil.Prometheus) {
+	e2eutil.ForeachPrometheus(t, func(t testing.TB, p *e2eutil.Prometheus) {
 		dir, err := ioutil.TempDir("", "shipper-e2e-test")
 		testutil.Ok(t, err)
 		defer func() {

diff --git a/pkg/store/bucket.go b/pkg/store/bucket.go
@@ -1703,7 +1703,7 @@ type partitioner interface {
 	// Partition partitions length entries into n <= length ranges that cover all
 	// input ranges
 	// It supports overlapping ranges.
-	// NOTE: It expects range to be sorted by start time.
+	// NOTE: It expects range to be ted by start time.
 	Partition(length int, rng func(int) (uint64, uint64)) []part
 }
 

diff --git a/pkg/store/bucket_e2e_test.go b/pkg/store/bucket_e2e_test.go
@@ -26,6 +26,7 @@ import (
 	storecache "github.com/thanos-io/thanos/pkg/store/cache"
 	"github.com/thanos-io/thanos/pkg/store/storepb"
 	"github.com/thanos-io/thanos/pkg/testutil"
+	"github.com/thanos-io/thanos/pkg/testutil/e2eutil"
 )
 
 var (
@@ -100,9 +101,9 @@ func prepareTestBlocks(t testing.TB, now time.Time, count int, dir string, bkt o
 
 		// Create two blocks per time slot. Only add 10 samples each so only one chunk
 		// gets created each. This way we can easily verify we got 10 chunks per series below.
-		id1, err := testutil.CreateBlock(ctx, dir, series[:4], 10, mint, maxt, extLset, 0)
+		id1, err := e2eutil.CreateBlock(ctx, dir, series[:4], 10, mint, maxt, extLset, 0)
 		testutil.Ok(t, err)
-		id2, err := testutil.CreateBlock(ctx, dir, series[4:], 10, mint, maxt, extLset, 0)
+		id2, err := e2eutil.CreateBlock(ctx, dir, series[4:], 10, mint, maxt, extLset, 0)
 		testutil.Ok(t, err)
 
 		dir1, dir2 := filepath.Join(dir, id1.String()), filepath.Join(dir, id2.String())

diff --git a/pkg/store/bucket_test.go b/pkg/store/bucket_test.go
@@ -35,6 +35,7 @@ import (
 	"github.com/thanos-io/thanos/pkg/objstore/inmem"
 	"github.com/thanos-io/thanos/pkg/store/storepb"
 	"github.com/thanos-io/thanos/pkg/testutil"
+	"github.com/thanos-io/thanos/pkg/testutil/e2eutil"
 	"gopkg.in/yaml.v2"
 )
 
@@ -502,19 +503,19 @@ func TestBucketStore_Sharding(t *testing.T) {
 	bkt := inmem.NewBucket()
 	series := []labels.Labels{labels.FromStrings("a", "1", "b", "1")}
 
-	id1, err := testutil.CreateBlock(ctx, dir, series, 10, 0, 1000, labels.Labels{{Name: "cluster", Value: "a"}, {Name: "region", Value: "r1"}}, 0)
+	id1, err := e2eutil.CreateBlock(ctx, dir, series, 10, 0, 1000, labels.Labels{{Name: "cluster", Value: "a"}, {Name: "region", Value: "r1"}}, 0)
 	testutil.Ok(t, err)
 	testutil.Ok(t, block.Upload(ctx, logger, bkt, filepath.Join(dir, id1.String())))
 
-	id2, err := testutil.CreateBlock(ctx, dir, series, 10, 1000, 2000, labels.Labels{{Name: "cluster", Value: "a"}, {Name: "region", Value: "r1"}}, 0)
+	id2, err := e2eutil.CreateBlock(ctx, dir, series, 10, 1000, 2000, labels.Labels{{Name: "cluster", Value: "a"}, {Name: "region", Value: "r1"}}, 0)
 	testutil.Ok(t, err)
 	testutil.Ok(t, block.Upload(ctx, logger, bkt, filepath.Join(dir, id2.String())))
 
-	id3, err := testutil.CreateBlock(ctx, dir, series, 10, 0, 1000, labels.Labels{{Name: "cluster", Value: "b"}, {Name: "region", Value: "r1"}}, 0)
+	id3, err := e2eutil.CreateBlock(ctx, dir, series, 10, 0, 1000, labels.Labels{{Name: "cluster", Value: "b"}, {Name: "region", Value: "r1"}}, 0)
 	testutil.Ok(t, err)
 	testutil.Ok(t, block.Upload(ctx, logger, bkt, filepath.Join(dir, id3.String())))
 
-	id4, err := testutil.CreateBlock(ctx, dir, series, 10, 0, 1000, labels.Labels{{Name: "cluster", Value: "a"}, {Name: "region", Value: "r2"}}, 0)
+	id4, err := e2eutil.CreateBlock(ctx, dir, series, 10, 0, 1000, labels.Labels{{Name: "cluster", Value: "a"}, {Name: "region", Value: "r2"}}, 0)
 	testutil.Ok(t, err)
 	testutil.Ok(t, block.Upload(ctx, logger, bkt, filepath.Join(dir, id4.String())))