block: bring back debug meta upload

Bring back thanos-io#5110 into our fork so that it would be possible to reconstruct the exact remote object storage state when the compactor does something bad. Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
vinted · Mar 16, 2023 · 7ff456e · 7ff456e
1 parent 4189d80
commit 7ff456e
Show file tree

Hide file tree

Showing 3 changed files with 15 additions and 8 deletions.
diff --git a/pkg/block/block.go b/pkg/block/block.go
@@ -9,6 +9,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"fmt"
 	"io"
 	"os"
 	"path"
@@ -145,6 +146,10 @@ func upload(ctx context.Context, logger log.Logger, bkt objstore.Bucket, bdir st
 		return errors.Wrap(err, "encode meta file")
 	}
 
+	if err := bkt.Upload(ctx, path.Join(DebugMetas, fmt.Sprintf("%s.json", id)), strings.NewReader(metaEncoded.String())); err != nil {
+		return cleanUp(logger, bkt, id, errors.Wrap(err, "upload debug meta file"))
+	}
+
 	if err := objstore.UploadDir(ctx, logger, bkt, filepath.Join(bdir, ChunksDirname), path.Join(id.String(), ChunksDirname), options...); err != nil {
 		return cleanUp(logger, bkt, id, errors.Wrap(err, "upload chunks"))
 	}

diff --git a/pkg/block/block_test.go b/pkg/block/block_test.go
@@ -139,8 +139,8 @@ func TestUpload(t *testing.T) {
 	{
 		// Full block.
 		testutil.Ok(t, Upload(ctx, log.NewNopLogger(), bkt, path.Join(tmpDir, "test", b1.String()), metadata.NoneFunc))
-		testutil.Equals(t, 3, len(bkt.Objects()))
-		testutil.Equals(t, 3751, len(bkt.Objects()[path.Join(b1.String(), ChunksDirname, "000001")]))
+		testutil.Equals(t, 4, len(bkt.Objects()))
+		testutil.Equals(t, 3727, len(bkt.Objects()[path.Join(b1.String(), ChunksDirname, "000001")]))
 		testutil.Equals(t, 401, len(bkt.Objects()[path.Join(b1.String(), IndexFilename)]))
 		testutil.Equals(t, 546, len(bkt.Objects()[path.Join(b1.String(), MetaFilename)]))
 
@@ -172,7 +172,7 @@ func TestUpload(t *testing.T) {
 		"files": [
 			{
 				"rel_path": "chunks/000001",
-				"size_bytes": 3751
+				"size_bytes": 3727
 			},
 			{
 				"rel_path": "index",
@@ -189,8 +189,8 @@ func TestUpload(t *testing.T) {
 	{
 		// Test Upload is idempotent.
 		testutil.Ok(t, Upload(ctx, log.NewNopLogger(), bkt, path.Join(tmpDir, "test", b1.String()), metadata.NoneFunc))
-		testutil.Equals(t, 3, len(bkt.Objects()))
-		testutil.Equals(t, 3751, len(bkt.Objects()[path.Join(b1.String(), ChunksDirname, "000001")]))
+		testutil.Equals(t, 4, len(bkt.Objects()))
+		testutil.Equals(t, 3727, len(bkt.Objects()[path.Join(b1.String(), ChunksDirname, "000001")]))
 		testutil.Equals(t, 401, len(bkt.Objects()[path.Join(b1.String(), IndexFilename)]))
 		testutil.Equals(t, 546, len(bkt.Objects()[path.Join(b1.String(), MetaFilename)]))
 	}
@@ -207,7 +207,7 @@ func TestUpload(t *testing.T) {
 		err = Upload(ctx, log.NewNopLogger(), bkt, path.Join(tmpDir, b2.String()), metadata.NoneFunc)
 		testutil.NotOk(t, err)
 		testutil.Equals(t, "empty external labels are not allowed for Thanos block.", err.Error())
-		testutil.Equals(t, 3, len(bkt.Objects()))
+		testutil.Equals(t, 4, len(bkt.Objects()))
 	}
 	{
 		// No external labels with UploadPromBlocks.
@@ -221,8 +221,8 @@ func TestUpload(t *testing.T) {
 		testutil.Ok(t, err)
 		err = UploadPromBlock(ctx, log.NewNopLogger(), bkt, path.Join(tmpDir, b2.String()), metadata.NoneFunc)
 		testutil.Ok(t, err)
-		testutil.Equals(t, 6, len(bkt.Objects()))
-		testutil.Equals(t, 3736, len(bkt.Objects()[path.Join(b2.String(), ChunksDirname, "000001")]))
+		testutil.Equals(t, 8, len(bkt.Objects()))
+		testutil.Equals(t, 3727, len(bkt.Objects()[path.Join(b2.String(), ChunksDirname, "000001")]))
 		testutil.Equals(t, 401, len(bkt.Objects()[path.Join(b2.String(), IndexFilename)]))
 		testutil.Equals(t, 525, len(bkt.Objects()[path.Join(b2.String(), MetaFilename)]))
 	}

diff --git a/pkg/testutil/e2eutil/prometheus.go b/pkg/testutil/e2eutil/prometheus.go
@@ -481,6 +481,8 @@ func createBlock(
 		}
 	}()
 
+	r := rand.New(rand.NewSource(int64(numSamples)))
+
 	var g errgroup.Group
 	var timeStepSize = (maxt - mint) / int64(numSamples+1)
 	var batchSize = len(series) / runtime.GOMAXPROCS(0)