Merge "[FAB-3399] Improve unit tests for common ledger"

common/ledger/blkstorage/fsblkstorage/block_serialization.go

@@ -17,8 +17,6 @@ limitations under the License.
 package fsblkstorage
 
 import (
-	"fmt"
-
 	"github.com/golang/protobuf/proto"
 	ledgerutil "github.com/hyperledger/fabric/common/ledger/util"
 	"github.com/hyperledger/fabric/protos/common"
@@ -60,15 +58,12 @@ func deserializeBlock(serializedBlockBytes []byte) (*common.Block, error) {
 	var err error
 	b := ledgerutil.NewBuffer(serializedBlockBytes)
 	if block.Header, err = extractHeader(b); err != nil {
-		fmt.Printf("h:%s\n", err)
 		return nil, err
 	}
 	if block.Data, _, err = extractData(b); err != nil {
-		fmt.Printf("d:%s\n", err)
 		return nil, err
 	}
 	if block.Metadata, err = extractMetadata(b); err != nil {
-		fmt.Printf("m:%s\n", err)
 		return nil, err
 	}
 	return block, nil
@@ -137,9 +132,6 @@ func addMetadataBytes(blockMetadata *common.BlockMetadata, buf *proto.Buffer) er
 	if err := buf.EncodeVarint(numItems); err != nil {
 		return err
 	}
-	if numItems == 0 {
-		return nil
-	}
 	for _, b := range blockMetadata.Metadata {
 		if err := buf.EncodeRawBytes(b); err != nil {
 			return err

common/ledger/blkstorage/fsblkstorage/block_stream.go

@@ -71,7 +71,6 @@ func newBlockfileStream(rootDir string, fileNum int, startOffset int64) (*blockf
 	}
 	var newPosition int64
 	if newPosition, err = file.Seek(startOffset, 0); err != nil {
-		// file.Seek does not raise an error - simply seeks to the new position
 		return nil, err
 	}
 	if newPosition != startOffset {
@@ -186,8 +185,7 @@ func (s *blockStream) nextBlockBytesAndPlacementInfo() ([]byte, *blockPlacementI
 	var blockPlacementInfo *blockPlacementInfo
 	var err error
 	if blockBytes, blockPlacementInfo, err = s.currentFileStream.nextBlockBytesAndPlacementInfo(); err != nil {
-		logger.Debugf("current file [%d]", s.currentFileNum)
-		logger.Debugf("blockbytes [%d]. Err:%s", len(blockBytes), err)
+		logger.Debugf("current file [%d] length of blockbytes [%d]. Err:%s", s.currentFileNum, len(blockBytes), err)
 		return nil, nil, err
 	}
 	logger.Debugf("blockbytes [%d] read from file [%d]", len(blockBytes), s.currentFileNum)

common/ledger/blkstorage/fsblkstorage/blockfile_mgr.go

@@ -32,7 +32,7 @@ import (
 	putil "github.com/hyperledger/fabric/protos/utils"
 )
 
-var logger = flogging.MustGetLogger("kvledger")
+var logger = flogging.MustGetLogger("fsblkstorage")
 
 const (
 	blockfilePrefix = "blockfile_"
@@ -221,10 +221,6 @@ func deriveBlockfilePath(rootDir string, suffixNum int) string {
 	return rootDir + "/" + blockfilePrefix + fmt.Sprintf("%06d", suffixNum)
 }
 
-func (mgr *blockfileMgr) open() error {
-	return mgr.currentFileWriter.open()
-}
-
 func (mgr *blockfileMgr) close() {
 	mgr.currentFileWriter.close()
 }

common/ledger/blkstorage/fsblkstorage/blockfile_mgr_test.go

@@ -43,6 +43,7 @@ func TestBlockfileMgrCrashDuringWriting(t *testing.T) {
 	testBlockfileMgrCrashDuringWriting(t, 10, 2, 1000, 1)
 	testBlockfileMgrCrashDuringWriting(t, 10, 2, 1000, 0)
 	testBlockfileMgrCrashDuringWriting(t, 0, 0, 1000, 10)
+	testBlockfileMgrCrashDuringWriting(t, 0, 5, 1000, 10)
 }
 
 func testBlockfileMgrCrashDuringWriting(t *testing.T, numBlocksBeforeCheckpoint int,
@@ -52,11 +53,24 @@ func testBlockfileMgrCrashDuringWriting(t *testing.T, numBlocksBeforeCheckpoint
 	ledgerid := "testLedger"
 	blkfileMgrWrapper := newTestBlockfileWrapper(env, ledgerid)
 	bg, gb := testutil.NewBlockGenerator(t, ledgerid, false)
+
+	// create all necessary blocks
+	totalBlocks := numBlocksBeforeCheckpoint + numBlocksAfterCheckpoint
+	allBlocks := []*common.Block{gb}
+	allBlocks = append(allBlocks, bg.NextTestBlocks(totalBlocks+1)...)
+
+	// identify the blocks that are to be added beforeCP, afterCP, and after restart
 	blocksBeforeCP := []*common.Block{}
-	if numBlocksBeforeCheckpoint > 0 {
-		blocksBeforeCP = append(blocksBeforeCP, gb)
+	blocksAfterCP := []*common.Block{}
+	if numBlocksBeforeCheckpoint != 0 {
+		blocksBeforeCP = allBlocks[0:numBlocksBeforeCheckpoint]
+	}
+	if numBlocksAfterCheckpoint != 0 {
+		blocksAfterCP = allBlocks[numBlocksBeforeCheckpoint : numBlocksBeforeCheckpoint+numBlocksAfterCheckpoint]
 	}
-	blocksBeforeCP = append(blocksBeforeCP, bg.NextTestBlocks(numBlocksBeforeCheckpoint-1)...)
+	blocksAfterRestart := allBlocks[numBlocksBeforeCheckpoint+numBlocksAfterCheckpoint:]
+
+	// add blocks before cp
 	blkfileMgrWrapper.addBlocks(blocksBeforeCP)
 	currentCPInfo := blkfileMgrWrapper.blockfileMgr.cpInfo
 	cpInfo1 := &checkpointInfo{
@@ -65,14 +79,7 @@ func testBlockfileMgrCrashDuringWriting(t *testing.T, numBlocksBeforeCheckpoint
 		currentCPInfo.isChainEmpty,
 		currentCPInfo.lastBlockNumber}
 
-	blocksAfterCP := []*common.Block{}
-	if numBlocksBeforeCheckpoint == 0 {
-		blocksAfterCP = append(blocksAfterCP, gb)
-		blocksAfterCP = append(blocksAfterCP, bg.NextTestBlocks(numBlocksAfterCheckpoint-1)...)
-	} else {
-		blocksAfterCP = bg.NextTestBlocks(numBlocksAfterCheckpoint)
-	}
-
+	// add blocks after cp
 	blkfileMgrWrapper.addBlocks(blocksAfterCP)
 	cpInfo2 := blkfileMgrWrapper.blockfileMgr.cpInfo
 
@@ -94,12 +101,7 @@ func testBlockfileMgrCrashDuringWriting(t *testing.T, numBlocksBeforeCheckpoint
 	testutil.AssertEquals(t, cpInfo3, cpInfo2)
 
 	// add fresh blocks after restart
-	blocksAfterRestart := bg.NextTestBlocks(2)
 	blkfileMgrWrapper.addBlocks(blocksAfterRestart)
-	allBlocks := []*common.Block{}
-	allBlocks = append(allBlocks, blocksBeforeCP...)
-	allBlocks = append(allBlocks, blocksAfterCP...)
-	allBlocks = append(allBlocks, blocksAfterRestart...)
 	testBlockfileMgrBlockIterator(t, blkfileMgrWrapper.blockfileMgr, 0, len(allBlocks)-1, allBlocks)
 }
 

common/ledger/blkstorage/fsblkstorage/blockindex.go

@@ -81,7 +81,7 @@ func (index *blockIndex) getLastBlockIndexed() (uint64, error) {
 	var blockNumBytes []byte
 	var err error
 	if blockNumBytes, err = index.db.Get(indexCheckpointKey); err != nil {
-		return 0, nil
+		return 0, err
 	}
 	if blockNumBytes == nil {
 		return 0, errIndexEmpty
@@ -252,7 +252,7 @@ func (index *blockIndex) getTxValidationCodeByTxID(txID string) (peer.TxValidati
 	if err != nil {
 		return peer.TxValidationCode(-1), err
 	} else if raw == nil {
-		return peer.TxValidationCode(-1), blkstorage.ErrAttrNotIndexed
+		return peer.TxValidationCode(-1), blkstorage.ErrNotFoundInIndex
 	} else if len(raw) != 1 {
 		return peer.TxValidationCode(-1), errors.New("Invalid value in indexItems")
 	}

common/ledger/blkstorage/fsblkstorage/blockindex_test.go

@@ -119,6 +119,7 @@ func testBlockIndexSync(t *testing.T, numBlocks int, numBlocksToIndex int, syncB
 }
 
 func TestBlockIndexSelectiveIndexing(t *testing.T) {
+	testBlockIndexSelectiveIndexing(t, []blkstorage.IndexableAttr{})
 	testBlockIndexSelectiveIndexing(t, []blkstorage.IndexableAttr{blkstorage.IndexableAttrBlockHash})
 	testBlockIndexSelectiveIndexing(t, []blkstorage.IndexableAttr{blkstorage.IndexableAttrBlockNum})
 	testBlockIndexSelectiveIndexing(t, []blkstorage.IndexableAttr{blkstorage.IndexableAttrTxID})

common/ledger/blkstorage/fsblkstorage/blocks_itr_test.go

@@ -52,10 +52,32 @@ func TestBlocksItrBlockingNext(t *testing.T) {
 	<-doneChan
 }
 
+func TestBlockItrClose(t *testing.T) {
+	env := newTestEnv(t, NewConf(testPath(), 0))
+	defer env.Cleanup()
+	blkfileMgrWrapper := newTestBlockfileWrapper(env, "testLedger")
+	defer blkfileMgrWrapper.close()
+	blkfileMgr := blkfileMgrWrapper.blockfileMgr
+
+	blocks := testutil.ConstructTestBlocks(t, 5)
+	blkfileMgrWrapper.addBlocks(blocks)
+
+	itr, err := blkfileMgr.retrieveBlocks(1)
+	testutil.AssertNoError(t, err, "")
+
+	bh, _ := itr.Next()
+	testutil.AssertNotNil(t, bh)
+	itr.Close()
+
+	bh, err = itr.Next()
+	testutil.AssertNoError(t, err, "")
+	testutil.AssertNil(t, bh)
+}
+
 func testIterateAndVerify(t *testing.T, itr *blocksItr, blocks []*common.Block, doneChan chan bool) {
 	blocksIterated := 0
 	for {
-		logger.Infof("blocksIterated: %v", blocksIterated)
+		t.Logf("blocksIterated: %v", blocksIterated)
 		bh, err := itr.Next()
 		testutil.AssertNoError(t, err, "")
 		testutil.AssertEquals(t, bh.(*blockHolder).GetBlock(), blocks[blocksIterated])

common/ledger/blkstorage/fsblkstorage/fs_blockstore_provider_test.go

@@ -24,7 +24,10 @@ import (
 	"github.com/hyperledger/fabric/common/ledger"
 	"github.com/hyperledger/fabric/common/ledger/blkstorage"
 	"github.com/hyperledger/fabric/common/ledger/testutil"
+	"github.com/hyperledger/fabric/core/ledger/util"
 	"github.com/hyperledger/fabric/protos/common"
+	"github.com/hyperledger/fabric/protos/peer"
+	"github.com/hyperledger/fabric/protos/utils"
 )
 
 func TestMultipleBlockStores(t *testing.T) {
@@ -35,7 +38,7 @@ func TestMultipleBlockStores(t *testing.T) {
 	store1, _ := provider.OpenBlockStore("ledger1")
 	defer store1.Shutdown()
 
-	store2, _ := provider.OpenBlockStore("ledger2")
+	store2, _ := provider.CreateBlockStore("ledger2")
 	defer store2.Shutdown()
 
 	blocks1 := testutil.ConstructTestBlocks(t, 5)
@@ -49,6 +52,8 @@ func TestMultipleBlockStores(t *testing.T) {
 	}
 	checkBlocks(t, blocks1, store1)
 	checkBlocks(t, blocks2, store2)
+	checkWithWrongInputs(t, store1, 5)
+	checkWithWrongInputs(t, store2, 10)
 }
 
 func checkBlocks(t *testing.T, expectedBlocks []*common.Block, store blkstorage.BlockStore) {
@@ -62,6 +67,58 @@ func checkBlocks(t *testing.T, expectedBlocks []*common.Block, store blkstorage.
 		block := blockHolder.(ledger.BlockHolder).GetBlock()
 		testutil.AssertEquals(t, block, expectedBlocks[i])
 	}
+
+	for blockNum := 0; blockNum < len(expectedBlocks); blockNum++ {
+		block := expectedBlocks[blockNum]
+		flags := util.TxValidationFlags(block.Metadata.Metadata[common.BlockMetadataIndex_TRANSACTIONS_FILTER])
+		retrievedBlock, _ := store.RetrieveBlockByNumber(uint64(blockNum))
+		testutil.AssertEquals(t, retrievedBlock, block)
+
+		retrievedBlock, _ = store.RetrieveBlockByHash(block.Header.Hash())
+		testutil.AssertEquals(t, retrievedBlock, block)
+
+		for txNum := 0; txNum < len(block.Data.Data); txNum++ {
+			txEnvBytes := block.Data.Data[txNum]
+			txEnv, _ := utils.GetEnvelopeFromBlock(txEnvBytes)
+			txid, err := extractTxID(txEnvBytes)
+			testutil.AssertNoError(t, err, "")
+
+			retrievedBlock, _ := store.RetrieveBlockByTxID(txid)
+			testutil.AssertEquals(t, retrievedBlock, block)
+
+			retrievedTxEnv, _ := store.RetrieveTxByID(txid)
+			testutil.AssertEquals(t, retrievedTxEnv, txEnv)
+
+			retrievedTxEnv, _ = store.RetrieveTxByBlockNumTranNum(uint64(blockNum), uint64(txNum))
+			testutil.AssertEquals(t, retrievedTxEnv, txEnv)
+
+			retrievedTxValCode, err := store.RetrieveTxValidationCodeByTxID(txid)
+			testutil.AssertNoError(t, err, "")
+			testutil.AssertEquals(t, retrievedTxValCode, flags.Flag(txNum))
+		}
+	}
+}
+
+func checkWithWrongInputs(t *testing.T, store blkstorage.BlockStore, numBlocks int) {
+	block, err := store.RetrieveBlockByHash([]byte("non-existent-hash"))
+	testutil.AssertNil(t, block)
+	testutil.AssertEquals(t, err, blkstorage.ErrNotFoundInIndex)
+
+	block, err = store.RetrieveBlockByTxID("non-existent-txid")
+	testutil.AssertNil(t, block)
+	testutil.AssertEquals(t, err, blkstorage.ErrNotFoundInIndex)
+
+	tx, err := store.RetrieveTxByID("non-existent-txid")
+	testutil.AssertNil(t, tx)
+	testutil.AssertEquals(t, err, blkstorage.ErrNotFoundInIndex)
+
+	tx, err = store.RetrieveTxByBlockNumTranNum(uint64(numBlocks+1), uint64(0))
+	testutil.AssertNil(t, tx)
+	testutil.AssertEquals(t, err, blkstorage.ErrNotFoundInIndex)
+
+	txCode, err := store.RetrieveTxValidationCodeByTxID("non-existent-txid")
+	testutil.AssertEquals(t, txCode, peer.TxValidationCode(-1))
+	testutil.AssertEquals(t, err, blkstorage.ErrNotFoundInIndex)
 }
 
 func TestBlockStoreProvider(t *testing.T) {

common/ledger/blkstorage/fsblkstorage/fs_blockstore_test.go

@@ -0,0 +1,40 @@
+/*
+Copyright IBM Corp. 2017 All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+		 http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package fsblkstorage
+
+import (
+	"testing"
+
+	"github.com/hyperledger/fabric/common/ledger/testutil"
+)
+
+func TestWrongBlockNumber(t *testing.T) {
+	env := newTestEnv(t, NewConf(testPath(), 0))
+	defer env.Cleanup()
+
+	provider := env.provider
+	store, _ := provider.OpenBlockStore("testLedger")
+	defer store.Shutdown()
+
+	blocks := testutil.ConstructTestBlocks(t, 5)
+	for i := 0; i < 3; i++ {
+		err := store.AddBlock(blocks[i])
+		testutil.AssertNoError(t, err, "")
+	}
+	err := store.AddBlock(blocks[4])
+	testutil.AssertError(t, err, "Error shold have been thrown when adding block number 4 while block number 3 is expected")
+}

common/ledger/blkstorage/fsblkstorage/pkg_test.go

@@ -23,12 +23,18 @@ import (
 	"os"
 	"testing"
 
+	"github.com/hyperledger/fabric/common/flogging"
 	"github.com/hyperledger/fabric/common/ledger/blkstorage"
 	"github.com/hyperledger/fabric/common/ledger/testutil"
 
 	"github.com/hyperledger/fabric/protos/common"
 )
 
+func TestMain(m *testing.M) {
+	flogging.SetModuleLevel("fsblkstorage", "debug")
+	os.Exit(m.Run())
+}
+
 func testPath() string {
 	if path, err := ioutil.TempDir("", "fsblkstorage-"); err != nil {
 		panic(err)