remove all cache dirs at the end of each run

Signed-off-by: Krasi Georgiev <8903888+krasi-georgiev@users.noreply.github.com>

cmd/thanos/downsample.go

@@ -137,6 +137,13 @@ func downsampleBucket(
 	if err := os.MkdirAll(dir, 0777); err != nil {
 		return errors.Wrap(err, "create dir")
 	}
+
+	defer func() {
+		if err := os.RemoveAll(dir); err != nil {
+			level.Error(logger).Log("msg", "failed to remove downsample cache directory", "path", dir, "err", err)
+		}
+	}()
+
 	var metas []*metadata.Meta
 
 	err := bkt.Iter(ctx, "", func(name string) error {

pkg/compact/compact.go

@@ -972,6 +972,11 @@ func NewBucketCompactor(
 
 // Compact runs compaction over bucket.
 func (c *BucketCompactor) Compact(ctx context.Context) error {
+	defer func() {
+		if err := os.RemoveAll(c.compactDir); err != nil {
+			level.Error(c.logger).Log("msg", "failed to remove compaction cache directory", "path", c.compactDir, "err", err)
+		}
+	}()
 	// Loop over bucket and compact until there's no work left.
 	for {
 		var (