From 4f6a4c2a9331216144c0763b800d2c6f0b9c9cc0 Mon Sep 17 00:00:00 2001
From: Sandeep Sukhani <sandeep.d.sukhani@gmail.com>
Date: Thu, 27 Aug 2020 17:03:05 +0530
Subject: [PATCH] cleanup boltdb files in queriers during startup/shutdown
 (#2558)

---
 pkg/storage/stores/shipper/downloads/table.go         |  8 +++++++-
 pkg/storage/stores/shipper/downloads/table_manager.go | 10 ++++++++++
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/pkg/storage/stores/shipper/downloads/table.go b/pkg/storage/stores/shipper/downloads/table.go
index 5c80443b2cc92..772b355fe40ca 100644
--- a/pkg/storage/stores/shipper/downloads/table.go
+++ b/pkg/storage/stores/shipper/downloads/table.go
@@ -184,8 +184,14 @@ func (t *Table) Close() {
 	defer t.dbsMtx.Unlock()
 
 	for name, db := range t.dbs {
+		dbPath := db.boltdb.Path()
+
 		if err := db.boltdb.Close(); err != nil {
-			level.Error(util.Logger).Log("msg", fmt.Errorf("failed to close file %s for table %s", name, t.name))
+			level.Error(util.Logger).Log("msg", fmt.Sprintf("failed to close file %s for table %s", name, t.name), "err", err)
+		}
+
+		if err := os.Remove(dbPath); err != nil {
+			level.Error(util.Logger).Log("msg", fmt.Sprintf("failed to remove file %s for table %s", name, t.name), "err", err)
 		}
 	}
 
diff --git a/pkg/storage/stores/shipper/downloads/table_manager.go b/pkg/storage/stores/shipper/downloads/table_manager.go
index bd69a923f9a8e..8957784cc7466 100644
--- a/pkg/storage/stores/shipper/downloads/table_manager.go
+++ b/pkg/storage/stores/shipper/downloads/table_manager.go
@@ -3,6 +3,7 @@ package downloads
 import (
 	"context"
 	"fmt"
+	"os"
 	"sync"
 	"time"
 
@@ -37,6 +38,15 @@ type TableManager struct {
 }
 
 func NewTableManager(cfg Config, boltIndexClient BoltDBIndexClient, storageClient StorageClient, registerer prometheus.Registerer) (*TableManager, error) {
+	// cleanup existing directory and re-create it since we do not use existing files in it.
+	if err := os.RemoveAll(cfg.CacheDir); err != nil {
+		return nil, err
+	}
+
+	if err := chunk_util.EnsureDirectory(cfg.CacheDir); err != nil {
+		return nil, err
+	}
+
 	ctx, cancel := context.WithCancel(context.Background())
 	tm := &TableManager{
 		cfg:             cfg,