block: bring back debug meta upload

Bring back thanos-io#5110 into our fork so that it would be possible to reconstruct the exact remote object storage state when the compactor does something bad. Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
vinted · Mar 6, 2023 · f3acf4d · f3acf4d
1 parent 110656f
commit f3acf4d
Show file tree

Hide file tree

Showing 3 changed files with 16 additions and 9 deletions.
diff --git a/pkg/block/block.go b/pkg/block/block.go
@@ -9,6 +9,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"fmt"
 	"io"
 	"os"
 	"path"
@@ -145,6 +146,10 @@ func upload(ctx context.Context, logger log.Logger, bkt objstore.Bucket, bdir st
 		return errors.Wrap(err, "encode meta file")
 	}
 
+	if err := bkt.Upload(ctx, path.Join(DebugMetas, fmt.Sprintf("%s.json", id)), strings.NewReader(metaEncoded.String())); err != nil {
+		return cleanUp(logger, bkt, id, errors.Wrap(err, "upload debug meta file"))
+	}
+
 	if err := objstore.UploadDir(ctx, logger, bkt, filepath.Join(bdir, ChunksDirname), path.Join(id.String(), ChunksDirname), options...); err != nil {
 		return cleanUp(logger, bkt, id, errors.Wrap(err, "upload chunks"))
 	}

diff --git a/pkg/block/block_test.go b/pkg/block/block_test.go
@@ -138,8 +138,8 @@ func TestUpload(t *testing.T) {
 	{
 		// Full block.
 		testutil.Ok(t, Upload(ctx, log.NewNopLogger(), bkt, path.Join(tmpDir, "test", b1.String()), metadata.NoneFunc))
-		testutil.Equals(t, 3, len(bkt.Objects()))
-		testutil.Equals(t, 3751, len(bkt.Objects()[path.Join(b1.String(), ChunksDirname, "000001")]))
+		testutil.Equals(t, 4, len(bkt.Objects()))
+		testutil.Equals(t, 3727, len(bkt.Objects()[path.Join(b1.String(), ChunksDirname, "000001")]))
 		testutil.Equals(t, 401, len(bkt.Objects()[path.Join(b1.String(), IndexFilename)]))
 		testutil.Equals(t, 546, len(bkt.Objects()[path.Join(b1.String(), MetaFilename)]))
 
@@ -171,7 +171,7 @@ func TestUpload(t *testing.T) {
 		"files": [
 			{
 				"rel_path": "chunks/000001",
-				"size_bytes": 3751
+				"size_bytes": 3727
 			},
 			{
 				"rel_path": "index",
@@ -188,8 +188,8 @@ func TestUpload(t *testing.T) {
 	{
 		// Test Upload is idempotent.
 		testutil.Ok(t, Upload(ctx, log.NewNopLogger(), bkt, path.Join(tmpDir, "test", b1.String()), metadata.NoneFunc))
-		testutil.Equals(t, 3, len(bkt.Objects()))
-		testutil.Equals(t, 3751, len(bkt.Objects()[path.Join(b1.String(), ChunksDirname, "000001")]))
+		testutil.Equals(t, 4, len(bkt.Objects()))
+		testutil.Equals(t, 3727, len(bkt.Objects()[path.Join(b1.String(), ChunksDirname, "000001")]))
 		testutil.Equals(t, 401, len(bkt.Objects()[path.Join(b1.String(), IndexFilename)]))
 		testutil.Equals(t, 546, len(bkt.Objects()[path.Join(b1.String(), MetaFilename)]))
 	}
@@ -206,7 +206,7 @@ func TestUpload(t *testing.T) {
 		err = Upload(ctx, log.NewNopLogger(), bkt, path.Join(tmpDir, b2.String()), metadata.NoneFunc)
 		testutil.NotOk(t, err)
 		testutil.Equals(t, "empty external labels are not allowed for Thanos block.", err.Error())
-		testutil.Equals(t, 3, len(bkt.Objects()))
+		testutil.Equals(t, 4, len(bkt.Objects()))
 	}
 	{
 		// No external labels with UploadPromBlocks.
@@ -220,8 +220,8 @@ func TestUpload(t *testing.T) {
 		testutil.Ok(t, err)
 		err = UploadPromBlock(ctx, log.NewNopLogger(), bkt, path.Join(tmpDir, b2.String()), metadata.NoneFunc)
 		testutil.Ok(t, err)
-		testutil.Equals(t, 6, len(bkt.Objects()))
-		testutil.Equals(t, 3736, len(bkt.Objects()[path.Join(b2.String(), ChunksDirname, "000001")]))
+		testutil.Equals(t, 8, len(bkt.Objects()))
+		testutil.Equals(t, 3727, len(bkt.Objects()[path.Join(b2.String(), ChunksDirname, "000001")]))
 		testutil.Equals(t, 401, len(bkt.Objects()[path.Join(b2.String(), IndexFilename)]))
 		testutil.Equals(t, 525, len(bkt.Objects()[path.Join(b2.String(), MetaFilename)]))
 	}

diff --git a/pkg/testutil/e2eutil/prometheus.go b/pkg/testutil/e2eutil/prometheus.go
@@ -440,6 +440,8 @@ func createBlock(
 		}
 	}()
 
+	r := rand.New(rand.NewSource(int64(numSamples)))
+
 	var g errgroup.Group
 	var timeStepSize = (maxt - mint) / int64(numSamples+1)
 	var batchSize = len(series) / runtime.GOMAXPROCS(0)
@@ -459,7 +461,7 @@ func createBlock(
 				app := h.Appender(ctx)
 
 				for _, lset := range batch {
-					_, err := app.Append(0, lset, t, rand.Float64())
+					_, err := app.Append(0, lset, t, r.Float64())
 					if err != nil {
 						if rerr := app.Rollback(); rerr != nil {
 							err = errors.Wrapf(err, "rollback failed: %v", rerr)