[FAB-18191] Remove contents of leveldb dir instead of the dir itself …

…when dropping dbs (#1828)

When dropping a leveldb, remove all contents of the directory instead of
the directory itself. This is common code for upgrade-dbs, rebuild, reset, and rollback.

Signed-off-by: Wenjian Qiao <wenjianq@gmail.com>

common/ledger/blkstorage/reset.go

@@ -56,8 +56,8 @@ func ResetBlockStore(blockStorageDir string) error {
 func DeleteBlockStoreIndex(blockStorageDir string) error {
 	conf := &Conf{blockStorageDir: blockStorageDir}
 	indexDir := conf.getIndexDir()
-	logger.Infof("Dropping the index dir [%s]... if present", indexDir)
-	return os.RemoveAll(indexDir)
+	logger.Infof("Dropping all contents under the index dir [%s]... if present", indexDir)
+	return fileutil.RemoveContents(indexDir)
 }
 
 func resetToGenesisBlk(ledgerDir string) error {

core/ledger/kvledger/drop_dbs.go

@@ -6,11 +6,7 @@ SPDX-License-Identifier: Apache-2.0
 
 package kvledger
 
-import (
-	"os"
-
-	"github.com/pkg/errors"
-)
+import "github.com/hyperledger/fabric/internal/fileutil"
 
 func dropDBs(rootFSPath string) error {
 	// During block commits to stateDB, the transaction manager updates the bookkeeperDB and one of the
@@ -36,26 +32,24 @@ func dropDBs(rootFSPath string) error {
 
 func dropStateLevelDB(rootFSPath string) error {
 	stateLeveldbPath := StateDBPath(rootFSPath)
-	logger.Infof("Dropping StateLevelDB at location [%s] ...if present", stateLeveldbPath)
-	return os.RemoveAll(stateLeveldbPath)
+	logger.Infof("Dropping all contents in StateLevelDB at location [%s] ...if present", stateLeveldbPath)
+	return fileutil.RemoveContents(stateLeveldbPath)
 }
 
 func dropConfigHistoryDB(rootFSPath string) error {
 	configHistoryDBPath := ConfigHistoryDBPath(rootFSPath)
-	logger.Infof("Dropping ConfigHistoryDB at location [%s]", configHistoryDBPath)
-	err := os.RemoveAll(configHistoryDBPath)
-	return errors.Wrapf(err, "error removing the ConfigHistoryDB located at %s", configHistoryDBPath)
+	logger.Infof("Dropping all contents in ConfigHistoryDB at location [%s] ...if present", configHistoryDBPath)
+	return fileutil.RemoveContents(configHistoryDBPath)
 }
 
 func dropBookkeeperDB(rootFSPath string) error {
 	bookkeeperDBPath := BookkeeperDBPath(rootFSPath)
-	logger.Infof("Dropping BookkeeperDB at location [%s]", bookkeeperDBPath)
-	err := os.RemoveAll(bookkeeperDBPath)
-	return errors.Wrapf(err, "error removing the BookkeeperDB located at %s", bookkeeperDBPath)
+	logger.Infof("Dropping all contents in BookkeeperDB at location [%s] ...if present", bookkeeperDBPath)
+	return fileutil.RemoveContents(bookkeeperDBPath)
 }
 
 func dropHistoryDB(rootFSPath string) error {
 	historyDBPath := HistoryDBPath(rootFSPath)
-	logger.Infof("Dropping HistoryDB at location [%s] ...if present", historyDBPath)
-	return os.RemoveAll(historyDBPath)
+	logger.Infof("Dropping all contents under in HistoryDB at location [%s] ...if present", historyDBPath)
+	return fileutil.RemoveContents(historyDBPath)
 }

core/ledger/kvledger/rebuild_dbs_test.go

@@ -7,12 +7,12 @@ SPDX-License-Identifier: Apache-2.0
 package kvledger
 
 import (
-	"os"
 	"path/filepath"
 	"testing"
 
 	configtxtest "github.com/hyperledger/fabric/common/configtx/test"
 	"github.com/hyperledger/fabric/core/ledger/mock"
+	"github.com/hyperledger/fabric/internal/fileutil"
 	"github.com/stretchr/testify/require"
 )
 
@@ -38,16 +38,21 @@ func TestRebuildDBs(t *testing.T) {
 
 	// verify blockstoreIndex, configHistory, history, state, bookkeeper dbs are deleted
 	rootFSPath := conf.RootFSPath
-	_, err = os.Stat(filepath.Join(BlockStorePath(rootFSPath), "index"))
-	require.Equal(t, os.IsNotExist(err), true)
-	_, err = os.Stat(ConfigHistoryDBPath(rootFSPath))
-	require.Equal(t, os.IsNotExist(err), true)
-	_, err = os.Stat(HistoryDBPath(rootFSPath))
-	require.Equal(t, os.IsNotExist(err), true)
-	_, err = os.Stat(StateDBPath(rootFSPath))
-	require.Equal(t, os.IsNotExist(err), true)
-	_, err = os.Stat(BookkeeperDBPath(rootFSPath))
-	require.Equal(t, os.IsNotExist(err), true)
+	empty, err := fileutil.DirEmpty(filepath.Join(BlockStorePath(rootFSPath), "index"))
+	require.NoError(t, err)
+	require.True(t, empty)
+	empty, err = fileutil.DirEmpty(ConfigHistoryDBPath(rootFSPath))
+	require.NoError(t, err)
+	require.True(t, empty)
+	empty, err = fileutil.DirEmpty(HistoryDBPath(rootFSPath))
+	require.NoError(t, err)
+	require.True(t, empty)
+	empty, err = fileutil.DirEmpty(StateDBPath(rootFSPath))
+	require.NoError(t, err)
+	require.True(t, empty)
+	empty, err = fileutil.DirEmpty(BookkeeperDBPath(rootFSPath))
+	require.NoError(t, err)
+	require.True(t, empty)
 
 	// rebuild again should be successful
 	err = RebuildDBs(conf)

core/ledger/kvledger/tests/env.go

@@ -79,7 +79,7 @@ func (e *env) cleanup() {
 	os.RemoveAll(e.initializer.Config.RootFSPath)
 }
 
-func (e *env) closeAllLedgersAndDrop(flags rebuildable) {
+func (e *env) closeAllLedgersAndRemoveDirContents(flags rebuildable) {
 	if e.ledgerMgr != nil {
 		e.ledgerMgr.Close()
 	}
@@ -89,38 +89,38 @@ func (e *env) closeAllLedgersAndDrop(flags rebuildable) {
 		indexPath := e.getBlockIndexDBPath()
 		logger.Infof("Deleting blockstore indexdb path [%s]", indexPath)
 		e.verifyNonEmptyDirExists(indexPath)
-		e.assert.NoError(os.RemoveAll(indexPath))
+		e.assert.NoError(fileutil.RemoveContents(indexPath))
 	}
 
 	if flags&rebuildableStatedb == rebuildableStatedb {
 		statedbPath := e.getLevelstateDBPath()
 		logger.Infof("Deleting statedb path [%s]", statedbPath)
 		e.verifyNonEmptyDirExists(statedbPath)
-		e.assert.NoError(os.RemoveAll(statedbPath))
+		e.assert.NoError(fileutil.RemoveContents(statedbPath))
 	}
 
 	if flags&rebuildableConfigHistory == rebuildableConfigHistory {
 		configHistoryPath := e.getConfigHistoryDBPath()
 		logger.Infof("Deleting configHistory db path [%s]", configHistoryPath)
 		e.verifyNonEmptyDirExists(configHistoryPath)
-		e.assert.NoError(os.RemoveAll(configHistoryPath))
+		e.assert.NoError(fileutil.RemoveContents(configHistoryPath))
 	}
 
 	if flags&rebuildableBookkeeper == rebuildableBookkeeper {
 		bookkeeperPath := e.getBookkeeperDBPath()
 		logger.Infof("Deleting bookkeeper db path [%s]", bookkeeperPath)
 		e.verifyNonEmptyDirExists(bookkeeperPath)
-		e.assert.NoError(os.RemoveAll(bookkeeperPath))
+		e.assert.NoError(fileutil.RemoveContents(bookkeeperPath))
 	}
 
 	if flags&rebuildableHistoryDB == rebuildableHistoryDB {
 		historyPath := e.getHistoryDBPath()
 		logger.Infof("Deleting history db path [%s]", historyPath)
 		e.verifyNonEmptyDirExists(historyPath)
-		e.assert.NoError(os.RemoveAll(historyPath))
+		e.assert.NoError(fileutil.RemoveContents(historyPath))
 	}
 
-	e.verifyRebuilableDoesNotExist(flags)
+	e.verifyRebuilableDirEmpty(flags)
 }
 
 func (e *env) verifyRebuilablesExist(flags rebuildable) {
@@ -141,21 +141,21 @@ func (e *env) verifyRebuilablesExist(flags rebuildable) {
 	}
 }
 
-func (e *env) verifyRebuilableDoesNotExist(flags rebuildable) {
+func (e *env) verifyRebuilableDirEmpty(flags rebuildable) {
 	if flags&rebuildableStatedb == rebuildableStatedb {
-		e.verifyDirDoesNotExist(e.getLevelstateDBPath())
+		e.verifyDirEmpty(e.getLevelstateDBPath())
 	}
 	if flags&rebuildableBlockIndex == rebuildableBlockIndex {
-		e.verifyDirDoesNotExist(e.getBlockIndexDBPath())
+		e.verifyDirEmpty(e.getBlockIndexDBPath())
 	}
 	if flags&rebuildableConfigHistory == rebuildableConfigHistory {
-		e.verifyDirDoesNotExist(e.getConfigHistoryDBPath())
+		e.verifyDirEmpty(e.getConfigHistoryDBPath())
 	}
 	if flags&rebuildableBookkeeper == rebuildableBookkeeper {
-		e.verifyDirDoesNotExist(e.getBookkeeperDBPath())
+		e.verifyDirEmpty(e.getBookkeeperDBPath())
 	}
 	if flags&rebuildableHistoryDB == rebuildableHistoryDB {
-		e.verifyDirDoesNotExist(e.getHistoryDBPath())
+		e.verifyDirEmpty(e.getHistoryDBPath())
 	}
 }
 
@@ -165,10 +165,10 @@ func (e *env) verifyNonEmptyDirExists(path string) {
 	e.assert.False(empty)
 }
 
-func (e *env) verifyDirDoesNotExist(path string) {
-	exists, _, err := fileutil.FileExists(path)
+func (e *env) verifyDirEmpty(path string) {
+	empty, err := fileutil.DirEmpty(path)
 	e.assert.NoError(err)
-	e.assert.False(exists)
+	e.assert.True(empty)
 }
 
 func (e *env) initLedgerMgmt() {

core/ledger/kvledger/tests/rebuild_test.go

@@ -26,7 +26,7 @@ func TestRebuildComponents(t *testing.T) {
 
 	t.Run("rebuild only statedb",
 		func(t *testing.T) {
-			env.closeAllLedgersAndDrop(rebuildableStatedb)
+			env.closeAllLedgersAndRemoveDirContents(rebuildableStatedb)
 			h1, h2 := env.newTestHelperOpenLgr("ledger1", t), env.newTestHelperOpenLgr("ledger2", t)
 			dataHelper.verifyLedgerContent(h1)
 			dataHelper.verifyLedgerContent(h2)
@@ -35,7 +35,7 @@ func TestRebuildComponents(t *testing.T) {
 
 	t.Run("rebuild statedb and config history",
 		func(t *testing.T) {
-			env.closeAllLedgersAndDrop(rebuildableStatedb | rebuildableConfigHistory)
+			env.closeAllLedgersAndRemoveDirContents(rebuildableStatedb | rebuildableConfigHistory)
 			h1, h2 := env.newTestHelperOpenLgr("ledger1", t), env.newTestHelperOpenLgr("ledger2", t)
 			dataHelper.verifyLedgerContent(h1)
 			dataHelper.verifyLedgerContent(h2)
@@ -44,7 +44,7 @@ func TestRebuildComponents(t *testing.T) {
 
 	t.Run("rebuild statedb and block index",
 		func(t *testing.T) {
-			env.closeAllLedgersAndDrop(rebuildableStatedb | rebuildableBlockIndex)
+			env.closeAllLedgersAndRemoveDirContents(rebuildableStatedb | rebuildableBlockIndex)
 			h1, h2 := env.newTestHelperOpenLgr("ledger1", t), env.newTestHelperOpenLgr("ledger2", t)
 			dataHelper.verifyLedgerContent(h1)
 			dataHelper.verifyLedgerContent(h2)
@@ -53,7 +53,7 @@ func TestRebuildComponents(t *testing.T) {
 
 	t.Run("rebuild statedb and historydb",
 		func(t *testing.T) {
-			env.closeAllLedgersAndDrop(rebuildableStatedb | rebuildableHistoryDB)
+			env.closeAllLedgersAndRemoveDirContents(rebuildableStatedb | rebuildableHistoryDB)
 			h1, h2 := env.newTestHelperOpenLgr("ledger1", t), env.newTestHelperOpenLgr("ledger2", t)
 			dataHelper.verifyLedgerContent(h1)
 			dataHelper.verifyLedgerContent(h2)
@@ -103,7 +103,7 @@ func TestRebuildComponentsWithBTL(t *testing.T) {
 	})
 
 	// rebuild statedb and bookkeeper
-	env.closeAllLedgersAndDrop(rebuildableStatedb | rebuildableBookkeeper)
+	env.closeAllLedgersAndRemoveDirContents(rebuildableStatedb | rebuildableBookkeeper)
 
 	h = env.newTestHelperOpenLgr("ledger1", t)
 	h.verifyPvtState("cc1", "coll1", "key1", "value1")                  // key1 should still exist in the state

core/ledger/kvledger/tests/reset_test.go

@@ -49,7 +49,7 @@ func TestResetAllLedgers(t *testing.T) {
 	err := kvledger.ResetAllKVLedgers(rootFSPath)
 	require.NoError(t, err)
 	rebuildable := rebuildableStatedb | rebuildableBookkeeper | rebuildableConfigHistory | rebuildableHistoryDB | rebuildableBlockIndex
-	env.verifyRebuilableDoesNotExist(rebuildable)
+	env.verifyRebuilableDirEmpty(rebuildable)
 	env.initLedgerMgmt()
 	preResetHt, err := kvledger.LoadPreResetHeight(rootFSPath, ledgerIDs)
 	require.NoError(t, err)
@@ -155,7 +155,7 @@ func TestResetAllLedgersWithBTL(t *testing.T) {
 	err := kvledger.ResetAllKVLedgers(env.initializer.Config.RootFSPath)
 	require.NoError(t, err)
 	rebuildable := rebuildableStatedb | rebuildableBookkeeper | rebuildableConfigHistory | rebuildableHistoryDB | rebuildableBlockIndex
-	env.verifyRebuilableDoesNotExist(rebuildable)
+	env.verifyRebuilableDirEmpty(rebuildable)
 	env.initLedgerMgmt()
 
 	// ensure that the reset is executed correctly
@@ -204,7 +204,7 @@ func TestResetLedgerWithoutDroppingDBs(t *testing.T) {
 	rebuildable := rebuildableStatedb | rebuildableBookkeeper | rebuildableConfigHistory | rebuildableHistoryDB
 	env.verifyRebuilablesExist(rebuildable)
 	rebuildable = rebuildableBlockIndex
-	env.verifyRebuilableDoesNotExist(rebuildable)
+	env.verifyRebuilableDirEmpty(rebuildable)
 	env.initLedgerMgmt()
 	preResetHt, err := kvledger.LoadPreResetHeight(env.initializer.Config.RootFSPath, []string{"ledger-1"})
 	t.Logf("preResetHt = %#v", preResetHt)

core/ledger/kvledger/tests/rollback_test.go

@@ -43,7 +43,7 @@ func TestRollbackKVLedger(t *testing.T) {
 	err = kvledger.RollbackKVLedger(env.initializer.Config.RootFSPath, "testLedger", targetBlockNum)
 	require.NoError(t, err)
 	rebuildable := rebuildableStatedb + rebuildableBookkeeper + rebuildableConfigHistory + rebuildableHistoryDB
-	env.verifyRebuilableDoesNotExist(rebuildable)
+	env.verifyRebuilableDirEmpty(rebuildable)
 	env.initLedgerMgmt()
 	preResetHt, err := kvledger.LoadPreResetHeight(env.initializer.Config.RootFSPath, []string{"testLedger"})
 	require.NoError(t, err)
@@ -117,7 +117,7 @@ func TestRollbackKVLedgerWithBTL(t *testing.T) {
 	err := kvledger.RollbackKVLedger(env.initializer.Config.RootFSPath, "ledger1", 4)
 	require.NoError(t, err)
 	rebuildable := rebuildableStatedb | rebuildableBookkeeper | rebuildableConfigHistory | rebuildableHistoryDB
-	env.verifyRebuilableDoesNotExist(rebuildable)
+	env.verifyRebuilableDirEmpty(rebuildable)
 
 	env.initLedgerMgmt()
 	h = env.newTestHelperOpenLgr("ledger1", t)

core/ledger/kvledger/tests/v1x_test.go

@@ -40,8 +40,9 @@ func TestV11(t *testing.T) {
 	require.NoError(t, testutil.Unzip("testdata/v11/sample_ledgers/ledgersData.zip", ledgerFSRoot, false))
 
 	require.NoError(t, kvledger.UpgradeDBs(env.initializer.Config))
-	rebuildable := rebuildableStatedb | rebuildableBookkeeper | rebuildableConfigHistory | rebuildableHistoryDB | rebuildableBlockIndex
-	env.verifyRebuilableDoesNotExist(rebuildable)
+	// do not include bookkeeper and confighistory dbs since the v11 ledger doesn't have these dbs
+	rebuildable := rebuildableStatedb | rebuildableHistoryDB | rebuildableBlockIndex
+	env.verifyRebuilableDirEmpty(rebuildable)
 
 	env.initLedgerMgmt()
 
@@ -54,7 +55,7 @@ func TestV11(t *testing.T) {
 	// rebuild and verify again
 	env.ledgerMgr.Close()
 	require.NoError(t, kvledger.RebuildDBs(env.initializer.Config))
-	env.verifyRebuilableDoesNotExist(rebuildable)
+	env.verifyRebuilableDirEmpty(rebuildable)
 	env.initLedgerMgmt()
 
 	h1, h2 = env.newTestHelperOpenLgr("ledger1", t), env.newTestHelperOpenLgr("ledger2", t)
@@ -171,8 +172,9 @@ func testV11CommitHashes(t *testing.T,
 	require.NoError(t, testutil.Unzip(v11DataPath, ledgerFSRoot, false))
 
 	require.NoError(t, kvledger.UpgradeDBs(env.initializer.Config))
-	rebuildable := rebuildableStatedb | rebuildableBookkeeper | rebuildableConfigHistory | rebuildableHistoryDB | rebuildableBlockIndex
-	env.verifyRebuilableDoesNotExist(rebuildable)
+	// do not include bookkeeper and confighistory dbs since the v11 ledger doesn't have these dbs
+	rebuildable := rebuildableStatedb | rebuildableHistoryDB | rebuildableBlockIndex
+	env.verifyRebuilableDirEmpty(rebuildable)
 
 	env.initLedgerMgmt()
 	h := env.newTestHelperOpenLgr("ledger1", t)
@@ -249,7 +251,7 @@ func TestV13WithStateCouchdb(t *testing.T) {
 	require.NoError(t, kvledger.UpgradeDBs(env.initializer.Config))
 	require.True(t, statecouchdb.IsEmpty(t, couchdbConfig))
 	rebuildable := rebuildableBookkeeper | rebuildableConfigHistory | rebuildableHistoryDB | rebuildableBlockIndex
-	env.verifyRebuilableDoesNotExist(rebuildable)
+	env.verifyRebuilableDirEmpty(rebuildable)
 
 	env.initLedgerMgmt()
 
@@ -262,7 +264,7 @@ func TestV13WithStateCouchdb(t *testing.T) {
 	env.ledgerMgr.Close()
 	require.NoError(t, kvledger.RebuildDBs(env.initializer.Config))
 	require.True(t, statecouchdb.IsEmpty(t, couchdbConfig))
-	env.verifyRebuilableDoesNotExist(rebuildable)
+	env.verifyRebuilableDirEmpty(rebuildable)
 	env.initLedgerMgmt()
 
 	h1, h2 = env.newTestHelperOpenLgr("ledger1", t), env.newTestHelperOpenLgr("ledger2", t)

internal/fileutil/fileutil.go

@@ -145,3 +145,22 @@ func ListSubdirs(dirPath string) ([]string, error) {
 	}
 	return subdirs, nil
 }
+
+// RemoveContents removes all the files and subdirs under the specified directory.
+// It returns nil if the specified directory does not exist.
+func RemoveContents(dir string) error {
+	contents, err := ioutil.ReadDir(dir)
+	if os.IsNotExist(err) {
+		return nil
+	}
+	if err != nil {
+		return errors.Wrapf(err, "error reading directory %s", dir)
+	}
+
+	for _, c := range contents {
+		if err = os.RemoveAll(filepath.Join(dir, c.Name())); err != nil {
+			return errors.Wrapf(err, "error removing %s under directory %s", c.Name(), dir)
+		}
+	}
+	return SyncDir(dir)
+}