[FAB-10095] Add pvt data related tests

This CR introduces a test folder under kvledger that contains
 - Util code for making it easy to write tests that operate
   at the level of ledger APIs

 - Tests related to pvt data functions utilizing the above mentioned
   util code

Change-Id: Ic16007cf2d6034249b0a5e10c0bcb0b800439dae
Signed-off-by: manish <manish.sethi@gmail.com>

core/ledger/kvledger/kv_ledger.go

@@ -140,6 +140,7 @@ func (l *kvLedger) recoverDBs() error {
 //recommitLostBlocks retrieves blocks in specified range and commit the write set to either
 //state DB or history DB or both
 func (l *kvLedger) recommitLostBlocks(firstBlockNum uint64, lastBlockNum uint64, recoverables ...recoverable) error {
+ logger.Debugf("recommitLostBlocks() - firstBlockNum=%d, lastBlockNum=%d, recoverables=%#v", firstBlockNum, lastBlockNum, recoverables)
  var err error
  var blockAndPvtdata *ledger.BlockAndPvtData
  for blockNumber := firstBlockNum; blockNumber <= lastBlockNum; blockNumber++ {

core/ledger/kvledger/tests/client.go

@@ -0,0 +1,125 @@
+/*
+Copyright IBM Corp. All Rights Reserved.
+
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package tests
+
+import (
+ "testing"
+
+ "github.com/golang/protobuf/proto"
+ "github.com/hyperledger/fabric/common/util"
+ "github.com/hyperledger/fabric/core/common/ccprovider"
+ "github.com/hyperledger/fabric/core/common/privdata"
+ "github.com/hyperledger/fabric/core/ledger"
+ "github.com/stretchr/testify/assert"
+)
+
+// client helps in a transction simulation. The client keeps accumlating the results of each simulated transaction
+// in a slice and at a later stage can be used to cut a test block for committing.
+// In a test, for each instantiated ledger, a single instance of a client is typically sufficient.
+type client struct {
+ lgr ledger.PeerLedger
+ simulatedTrans []*txAndPvtdata // accumulates the results of transactions simulations
+ assert *assert.Assertions
+}
+
+func newClient(lgr ledger.PeerLedger, t *testing.T) *client {
+ return &client{lgr, nil, assert.New(t)}
+}
+
+// simulateDataTx takes a simulation logic and wraps it between
+// (A) the pre-simulation tasks (such as obtaining a fresh simulator) and
+// (B) the post simulation tasks (such as gathering (public and pvt) simulation results and constructing a transaction)
+// Since (A) and (B) both are handled in this function, the test code can be kept simple by just supplying the simulation logic
+func (c *client) simulateDataTx(txid string, simulationLogic func(s *simulator)) *txAndPvtdata {
+ if txid == "" {
+ txid = util.GenerateUUID()
+ }
+ ledgerSimulator, err := c.lgr.NewTxSimulator(txid)
+ c.assert.NoError(err)
+ sim := &simulator{ledgerSimulator, txid, c.assert}
+ simulationLogic(sim)
+ txAndPvtdata := sim.done()
+ c.simulatedTrans = append(c.simulatedTrans, txAndPvtdata)
+ return txAndPvtdata
+}
+
+// simulateDeployTx mimics a transction that deploys a chaincode. This in turn calls the function 'simulateDataTx'
+// with supplying the simulation logic that mimics the inoke funciton of 'lscc' for the ledger tests
+func (c *client) simulateDeployTx(ccName string, collConfs []*collConf) *txAndPvtdata {
+ ccData := &ccprovider.ChaincodeData{Name: ccName}
+ ccDataBytes, err := proto.Marshal(ccData)
+ c.assert.NoError(err)
+
+ psudoLSCCInvokeFunc := func(s *simulator) {
+ s.setState("lscc", ccName, string(ccDataBytes))
+ if collConfs != nil {
+ protoBytes, err := convertToCollConfigProtoBytes(collConfs)
+ c.assert.NoError(err)
+ s.setState("lscc", privdata.BuildCollectionKVSKey(ccName), string(protoBytes))
+ }
+ }
+ return c.simulateDataTx("", psudoLSCCInvokeFunc)
+}
+
+// simulateUpgradeTx see comments on function 'simulateDeployTx'
+func (c *client) simulateUpgradeTx(ccName string, collConfs []*collConf) *txAndPvtdata {
+ return c.simulateDeployTx(ccName, collConfs)
+}
+
+/////////////////////// simulator wrapper functions ///////////////////////
+type simulator struct {
+ ledger.TxSimulator
+ txid string
+ assert *assert.Assertions
+}
+
+func (s *simulator) getState(ns, key string) string {
+ val, err := s.GetState(ns, key)
+ s.assert.NoError(err)
+ return string(val)
+}
+
+func (s *simulator) setState(ns, key string, val string) {
+ s.assert.NoError(
+ s.SetState(ns, key, []byte(val)),
+ )
+}
+
+func (s *simulator) delState(ns, key string) {
+ s.assert.NoError(
+ s.DeleteState(ns, key),
+ )
+}
+
+func (s *simulator) getPvtdata(ns, coll, key string) {
+ _, err := s.GetPrivateData(ns, coll, key)
+ s.assert.NoError(err)
+}
+
+func (s *simulator) setPvtdata(ns, coll, key string, val string) {
+ s.assert.NoError(
+ s.SetPrivateData(ns, coll, key, []byte(val)),
+ )
+}
+
+func (s *simulator) delPvtdata(ns, coll, key string) {
+ s.assert.NoError(
+ s.DeletePrivateData(ns, coll, key),
+ )
+}
+
+func (s *simulator) done() *txAndPvtdata {
+ s.Done()
+ simRes, err := s.GetTxSimulationResults()
+ s.assert.NoError(err)
+ pubRwsetBytes, err := simRes.GetPubSimulationBytes()
+ s.assert.NoError(err)
+ envelope, err := constructTransaction(s.txid, pubRwsetBytes)
+ s.assert.NoError(err)
+ txAndPvtdata := &txAndPvtdata{Txid: s.txid, Envelope: envelope, Pvtws: simRes.PvtSimulationResults}
+ return txAndPvtdata
+}

core/ledger/kvledger/tests/committer.go

@@ -0,0 +1,94 @@
+/*
+Copyright IBM Corp. All Rights Reserved.
+
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package tests
+
+import (
+ "testing"
+
+ "github.com/golang/protobuf/proto"
+ "github.com/hyperledger/fabric/core/ledger"
+ "github.com/hyperledger/fabric/protos/common"
+ "github.com/stretchr/testify/assert"
+)
+
+// committer helps in cutting a block and commits the block (with pvt data) to the ledger
+type committer struct {
+ lgr ledger.PeerLedger
+ blkgen *blkGenerator
+ assert *assert.Assertions
+}
+
+func newCommitter(lgr ledger.PeerLedger, t *testing.T) *committer {
+ return &committer{lgr, newBlockGenerator(lgr, t), assert.New(t)}
+}
+
+// cutBlockAndCommitWithPvtdata cuts the next block from the given 'txAndPvtdata' and commits the block (with pvt data) to the ledger
+// This function return a copy of 'ledger.BlockAndPvtData' that was submitted to the ledger to commit.
+// A copy is returned instead of the actual one because, ledger makes some changes to the submitted block before commit
+// (such as setting the metadata) and the test code would want to have the exact copy of the block that was submitted to
+// the ledger
+func (c *committer) cutBlockAndCommitWithPvtdata(trans ...*txAndPvtdata) *ledger.BlockAndPvtData {
+ blk := c.blkgen.nextBlockAndPvtdata(trans...)
+ blkCopy := c.copyOfBlockAndPvtdata(blk)
+ c.assert.NoError(
+ c.lgr.CommitWithPvtData(blk),
+ )
+ return blkCopy
+}
+
+func (c *committer) cutBlockAndCommitExpectError(trans ...*txAndPvtdata) (*ledger.BlockAndPvtData, error) {
+ blk := c.blkgen.nextBlockAndPvtdata(trans...)
+ blkCopy := c.copyOfBlockAndPvtdata(blk)
+ err := c.lgr.CommitWithPvtData(blk)
+ c.assert.Error(err)
+ return blkCopy, err
+}
+
+func (c *committer) copyOfBlockAndPvtdata(blk *ledger.BlockAndPvtData) *ledger.BlockAndPvtData {
+ blkBytes, err := proto.Marshal(blk.Block)
+ c.assert.NoError(err)
+ blkCopy := &common.Block{}
+ c.assert.NoError(proto.Unmarshal(blkBytes, blkCopy))
+ return &ledger.BlockAndPvtData{Block: blkCopy, BlockPvtData: blk.BlockPvtData}
+}
+
+///////////////// block generation code ///////////////////////////////////////////
+// blkGenerator helps creating the next block for the ledger
+type blkGenerator struct {
+ lastNum uint64
+ lastHash []byte
+ assert *assert.Assertions
+}
+
+// newBlockGenerator constructs a 'blkGenerator' and initializes the 'blkGenerator'
+// from the last block available in the ledger so that the next block can be populated
+// with the correct block number and previous block hash
+func newBlockGenerator(lgr ledger.PeerLedger, t *testing.T) *blkGenerator {
+ assert := assert.New(t)
+ info, err := lgr.GetBlockchainInfo()
+ assert.NoError(err)
+ return &blkGenerator{info.Height - 1, info.PreviousBlockHash, assert}
+}
+
+// nextBlockAndPvtdata cuts the next block
+func (g *blkGenerator) nextBlockAndPvtdata(trans ...*txAndPvtdata) *ledger.BlockAndPvtData {
+ block := common.NewBlock(g.lastNum+1, g.lastHash)
+ blockPvtdata := make(map[uint64]*ledger.TxPvtData)
+ for i, tran := range trans {
+ seq := uint64(i)
+ envelopeBytes, _ := proto.Marshal(tran.Envelope)
+ block.Data.Data = append(block.Data.Data, envelopeBytes)
+ if tran.Pvtws != nil {
+ blockPvtdata[seq] = &ledger.TxPvtData{SeqInBlock: seq, WriteSet: tran.Pvtws}
+ }
+ }
+ block.Header.DataHash = block.Data.Hash()
+ g.lastNum++
+ g.lastHash = block.Header.Hash()
+ setBlockFlagsToValid(block)
+ return &ledger.BlockAndPvtData{Block: block, BlockPvtData: blockPvtdata}
+}

core/ledger/kvledger/tests/env.go

@@ -0,0 +1,151 @@
+/*
+Copyright IBM Corp. All Rights Reserved.
+
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package tests
+
+import (
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/hyperledger/fabric/common/ledger/blkstorage/fsblkstorage"
+ "github.com/hyperledger/fabric/common/ledger/util"
+ "github.com/hyperledger/fabric/core/ledger/ledgerconfig"
+ "github.com/hyperledger/fabric/core/ledger/ledgermgmt"
+ "github.com/hyperledger/fabric/core/peer"
+ "github.com/spf13/viper"
+ "github.com/stretchr/testify/assert"
+)
+
+type config map[string]interface{}
+type rebuildable uint8
+
+const (
+ rebuildableStatedb rebuildable = 1
+ rebuildableBlockIndex rebuildable = 2
+ rebuildableConfigHistory rebuildable = 4
+ rebuildableHistoryDB rebuildable = 8
+)
+
+var (
+ defaultConfig = config{
+ "peer.fileSystemPath": "/tmp/fabric/ledgertests",
+ "ledger.state.stateDatabase": "goleveldb",
+ }
+)
+
+type env struct {
+ assert *assert.Assertions
+}
+
+func newEnv(conf config, t *testing.T) *env {
+ setupConfigs(conf)
+ env := &env{assert.New(t)}
+ initLedgerMgmt()
+ return env
+}
+
+func (e *env) cleanup() {
+ closeLedgerMgmt()
+ e.assert.NoError(os.RemoveAll(getLedgerRootPath()))
+}
+
+func (e *env) closeAllLedgersAndDrop(flags rebuildable) {
+ closeLedgerMgmt()
+ defer initLedgerMgmt()
+
+ if flags&rebuildableBlockIndex == rebuildableBlockIndex {
+ indexPath := getBlockIndexDBPath()
+ logger.Infof("Deleting blockstore indexdb path [%s]", indexPath)
+ e.verifyNonEmptyDirExists(indexPath)
+ e.assert.NoError(os.RemoveAll(indexPath))
+ }
+
+ if flags&rebuildableStatedb == rebuildableStatedb {
+ statedbPath := getLevelstateDBPath()
+ logger.Infof("Deleting statedb path [%s]", statedbPath)
+ e.verifyNonEmptyDirExists(statedbPath)
+ e.assert.NoError(os.RemoveAll(statedbPath))
+ }
+
+ if flags&rebuildableConfigHistory == rebuildableConfigHistory {
+ configHistory := getConfigHistoryDBPath()
+ logger.Infof("Deleting configHistory db path [%s]", configHistory)
+ e.verifyNonEmptyDirExists(configHistory)
+ e.assert.NoError(os.RemoveAll(configHistory))
+ }
+}
+
+func (e *env) verifyRebuilablesExist(flags rebuildable) {
+ if flags&rebuildableStatedb == rebuildableBlockIndex {
+ e.verifyNonEmptyDirExists(getBlockIndexDBPath())
+ }
+ if flags&rebuildableBlockIndex == rebuildableStatedb {
+ e.verifyNonEmptyDirExists(getLevelstateDBPath())
+ }
+ if flags&rebuildableConfigHistory == rebuildableConfigHistory {
+ e.verifyNonEmptyDirExists(getConfigHistoryDBPath())
+ }
+}
+
+func (e *env) verifyRebuilableDoesNotExist(flags rebuildable) {
+ if flags&rebuildableStatedb == rebuildableStatedb {
+ e.verifyDirDoesNotExist(getLevelstateDBPath())
+ }
+ if flags&rebuildableStatedb == rebuildableBlockIndex {
+ e.verifyDirDoesNotExist(getBlockIndexDBPath())
+ }
+ if flags&rebuildableConfigHistory == rebuildableConfigHistory {
+ e.verifyDirDoesNotExist(getConfigHistoryDBPath())
+ }
+}
+
+func (e *env) verifyNonEmptyDirExists(path string) {
+ empty, err := util.DirEmpty(path)
+ e.assert.NoError(err)
+ e.assert.False(empty)
+}
+
+func (e *env) verifyDirDoesNotExist(path string) {
+ exists, _, err := util.FileExists(path)
+ e.assert.NoError(err)
+ e.assert.False(exists)
+}
+
+// ########################### ledgermgmt and ledgerconfig related functions wrappers #############################
+// In the current code, ledgermgmt and ledgerconfigs are packaged scope APIs and hence so are the following
+// wrapper APIs. As a TODO, both the ledgermgmt and ledgerconfig can be refactored as separate objects and then
+// the instances of these two would be wrapped inside the `env` struct above.
+// #################################################################################################################
+func setupConfigs(conf config) {
+ for c, v := range conf {
+ viper.Set(c, v)
+ }
+}
+
+func initLedgerMgmt() {
+ ledgermgmt.InitializeExistingTestEnvWithCustomProcessors(peer.ConfigTxProcessors)
+}
+
+func closeLedgerMgmt() {
+ ledgermgmt.Close()
+}
+
+func getLedgerRootPath() string {
+ return ledgerconfig.GetRootPath()
+}
+
+func getLevelstateDBPath() string {
+ return ledgerconfig.GetStateLevelDBPath()
+}
+
+func getBlockIndexDBPath() string {
+ return filepath.Join(ledgerconfig.GetBlockStorePath(), fsblkstorage.IndexDir)
+}
+
+func getConfigHistoryDBPath() string {
+ return ledgerconfig.GetConfigHistoryPath()
+}