Contain the dependency on bccsp

This commit
- Removes the dependency on bccsp package from much deeper code and
contains the dependency only at the level of top level - kvledger

- Though the refactoring is straight forward, still this commit adds
a test to make sure that the code that generates and validates the
merkle tree for the range queries is backward compatible. This test
was long due that can help catch any incompatible changes in the code or
the hash function used

Signed-off-by: manish <manish.sethi@gmail.com>

core/ledger/kvledger/history/pkg_test.go

@@ -7,11 +7,11 @@ SPDX-License-Identifier: Apache-2.0
 package history
 
 import (
+	"crypto/sha256"
 	"io/ioutil"
 	"os"
 	"testing"
 
-	"github.com/hyperledger/fabric/bccsp/sw"
 	"github.com/hyperledger/fabric/common/ledger/blkstorage"
 	"github.com/hyperledger/fabric/common/metrics/disabled"
 	"github.com/hyperledger/fabric/core/ledger/kvledger/bookkeeping"
@@ -22,7 +22,15 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-/////// levelDBLockBasedHistoryEnv //////
+var (
+	testHashFunc = func(data []byte) ([]byte, error) {
+		h := sha256.New()
+		if _, err := h.Write(data); err != nil {
+			return nil, err
+		}
+		return h.Sum(nil), nil
+	}
+)
 
 type levelDBLockBasedHistoryEnv struct {
 	t                     testing.TB
@@ -50,8 +58,6 @@ func newTestHistoryEnv(t *testing.T) *levelDBLockBasedHistoryEnv {
 		t.Fatalf("Failed to create history database directory: %s", err)
 	}
 
-	cryptoProvider, err := sw.NewDefaultSecurityLevelWithKeystore(sw.NewDummyKeyStore())
-	assert.NoError(t, err)
 	txmgrInitializer := &lockbasedtxmgr.Initializer{
 		LedgerID:            testLedgerID,
 		DB:                  testDB,
@@ -60,7 +66,7 @@ func newTestHistoryEnv(t *testing.T) *levelDBLockBasedHistoryEnv {
 		BookkeepingProvider: testBookkeepingEnv.TestProvider,
 		CCInfoProvider:      &mock.DeployedChaincodeInfoProvider{},
 		CustomTxProcessors:  nil,
-		Hasher:              cryptoProvider,
+		HashFunc:            testHashFunc,
 	}
 	txMgr, err := lockbasedtxmgr.NewLockBasedTxMgr(txmgrInitializer)
 

core/ledger/kvledger/kv_ledger.go

@@ -15,6 +15,7 @@ import (
 	"github.com/golang/protobuf/proto"
 	"github.com/hyperledger/fabric-protos-go/common"
 	"github.com/hyperledger/fabric-protos-go/peer"
+	"github.com/hyperledger/fabric/bccsp"
 	"github.com/hyperledger/fabric/common/flogging"
 	commonledger "github.com/hyperledger/fabric/common/ledger"
 	"github.com/hyperledger/fabric/common/ledger/blkstorage"
@@ -36,6 +37,10 @@ import (
 
 var logger = flogging.MustGetLogger("kvledger")
 
+var (
+	rwsetHashOpts = &bccsp.SHA256Opts{}
+)
+
 // kvLedger provides an implementation of `ledger.PeerLedger`.
 // This implementation provides a key-value based data model
 type kvLedger struct {
@@ -91,7 +96,9 @@ func newKVLedger(initializer *lgrInitializer) (*kvLedger, error) {
 		BookkeepingProvider: initializer.bookkeeperProvider,
 		CCInfoProvider:      initializer.ccInfoProvider,
 		CustomTxProcessors:  initializer.customTxProcessors,
-		Hasher:              initializer.hasher,
+		HashFunc: func(data []byte) (hashsum []byte, err error) {
+			return initializer.hasher.Hash(data, rwsetHashOpts)
+		},
 	}
 	if err := l.initTxMgr(txmgrInitializer); err != nil {
 		return nil, err

core/ledger/kvledger/rwset_backward_compatibility_test.go

@@ -0,0 +1,97 @@
+/*
+Copyright IBM Corp. All Rights Reserved.
+
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package kvledger
+
+import (
+	"fmt"
+	"io/ioutil"
+	"testing"
+
+	"github.com/hyperledger/fabric-protos-go/common"
+	"github.com/hyperledger/fabric/common/ledger/testutil"
+	"github.com/hyperledger/fabric/common/util"
+	lgr "github.com/hyperledger/fabric/core/ledger"
+	"github.com/hyperledger/fabric/core/ledger/mock"
+	"github.com/hyperledger/fabric/protoutil"
+	"github.com/stretchr/testify/require"
+)
+
+// TestBackwardCompatibilityRWSetV21 is added to protect against any changes in the hash function
+// that is used in preparing the rwset that includes a merkle tree for the range query
+func TestBackwardCompatibilityRWSetV21(t *testing.T) {
+	rwsetBytes, err := ioutil.ReadFile("testdata/rwsetbytes_v21")
+	require.NoError(t, err)
+	b := testGenerateSampleRWSet(t)
+	require.Equal(t, rwsetBytes, b)
+}
+
+// TestGenerateSampleRWSet generates the rwset that includes a merkle tree for a range-query read-set as well
+// The data present in the file testdata/rwsetbytes_v21 is generated by running the below test code on version 2.1
+// To regenrate the data, (if needed in order to add some more data to the generated rwset), checkout release-2.1,
+// and uncomment and run the following test
+// func TestGenerateSampleRWSet(t *testing.T) {
+// 	b := testGenerateSampleRWSet(t)
+// 	require.NoError(t, ioutil.WriteFile("testdata/rwsetbytes_v21", b, 0644))
+// }
+
+func testGenerateSampleRWSet(t *testing.T) []byte {
+	conf, cleanup := testConfig(t)
+	defer cleanup()
+	provider := testutilNewProvider(conf, t, &mock.DeployedChaincodeInfoProvider{})
+	defer provider.Close()
+
+	bg, gb := testutil.NewBlockGenerator(t, "testLedger", false)
+	gbHash := protoutil.BlockHeaderHash(gb.Header)
+	ledger, err := provider.Create(gb)
+	require.NoError(t, err)
+	defer ledger.Close()
+
+	bcInfo, err := ledger.GetBlockchainInfo()
+	require.NoError(t, err)
+	require.Equal(t, &common.BlockchainInfo{
+		Height: 1, CurrentBlockHash: gbHash, PreviousBlockHash: nil,
+	}, bcInfo)
+
+	txid := util.GenerateUUID()
+
+	// perform a range query for significant larger scan so that the merkle tree building kicks in
+	// each level contains max 50 nodes per the current configuration
+	simulator, err := ledger.NewTxSimulator(txid)
+	for i := 0; i < 10011; i++ {
+		simulator.SetState("ns1", fmt.Sprintf("key-%000d", i), []byte(fmt.Sprintf("value-%000d", i)))
+	}
+	simulator.Done()
+	simRes, err := simulator.GetTxSimulationResults()
+	require.NoError(t, err)
+	pubSimBytes, err := simRes.GetPubSimulationBytes()
+	require.NoError(t, err)
+	block1 := bg.NextBlock([][]byte{pubSimBytes})
+	ledger.CommitLegacy(&lgr.BlockAndPvtData{Block: block1}, &lgr.CommitOptions{})
+
+	simulator, err = ledger.NewTxSimulator(txid)
+	require.NoError(t, err)
+	simulator.GetState("ns1", fmt.Sprintf("key-%000d", 5))
+	simulator.SetState("ns1", fmt.Sprintf("key-%000d", 6), []byte(fmt.Sprintf("value-%000d-new", 6)))
+	itr, err := simulator.GetStateRangeScanIterator("ns1", "", "")
+	require.NoError(t, err)
+	numKVs := 0
+	for {
+		kv, err := itr.Next()
+		require.NoError(t, err)
+		if kv == nil {
+			break
+		}
+		numKVs++
+	}
+	require.Equal(t, 10011, numKVs)
+	simulator.Done()
+	simRes, err = simulator.GetTxSimulationResults()
+	require.NoError(t, err)
+	pubSimBytes, err = simRes.GetPubSimulationBytes()
+	require.NoError(t, err)
+	return pubSimBytes
+}

core/ledger/kvledger/testdata/rwsetbytes_v21

@@ -0,0 +1,5 @@
+�

+ns1�

+
+key-5
�

*�
2 曮��-�^�c�6&�ǫ����7��E�� F������ќ;1CA���T�:k�j�U� ��ڼL� ؈�fw{����~��`�,Hž�� ����W?S��N�m�<�>��8J��y�\4�z�
+key-6value-6-new

core/ledger/kvledger/txmgmt/rwsetutil/query_results_helper.go

@@ -11,8 +11,6 @@ import (
 
 	"github.com/golang/protobuf/proto"
 	"github.com/hyperledger/fabric-protos-go/ledger/rwset/kvrwset"
-	"github.com/hyperledger/fabric/bccsp"
-	"github.com/hyperledger/fabric/core/ledger"
 	"github.com/pkg/errors"
 )
 
@@ -26,9 +24,9 @@ const (
 	leafLevel = MerkleTreeLevel(1)
 )
 
-var (
-	hashOpts = &bccsp.SHA256Opts{}
-)
+// HashFunc - the function signature for the hash function that is used in building and validating
+// the merkle tree in the rwset, for the range query results
+type HashFunc func(data []byte) (hashsum []byte, err error)
 
 // RangeQueryResultsHelper helps preparing range query results for phantom items detection during validation.
 // The results are expected to be fed as they are being iterated over.
@@ -55,21 +53,21 @@ type RangeQueryResultsHelper struct {
 	mt             *merkleTree
 	maxDegree      uint32
 	hashingEnabled bool
-	hasher         ledger.Hasher
+	hashFunc       HashFunc
 }
 
 // NewRangeQueryResultsHelper constructs a RangeQueryResultsHelper
-func NewRangeQueryResultsHelper(enableHashing bool, maxDegree uint32, hasher ledger.Hasher) (*RangeQueryResultsHelper, error) {
+func NewRangeQueryResultsHelper(enableHashing bool, maxDegree uint32, hashFunc HashFunc) (*RangeQueryResultsHelper, error) {
 	helper := &RangeQueryResultsHelper{
 		pendingResults: nil,
 		hashingEnabled: enableHashing,
 		maxDegree:      maxDegree,
 		mt:             nil,
-		hasher:         hasher,
+		hashFunc:       hashFunc,
 	}
 	if enableHashing {
 		var err error
-		if helper.mt, err = newMerkleTree(maxDegree, hasher); err != nil {
+		if helper.mt, err = newMerkleTree(maxDegree, hashFunc); err != nil {
 			return nil, err
 		}
 	}
@@ -130,7 +128,7 @@ func (helper *RangeQueryResultsHelper) processPendingResults() error {
 		return err
 	}
 	helper.pendingResults = nil
-	hash, err := helper.hasher.Hash(b, hashOpts)
+	hash, err := helper.hashFunc(b)
 	if err != nil {
 		return err
 	}
@@ -148,18 +146,18 @@ type merkleTree struct {
 	tree      map[MerkleTreeLevel][]Hash
 	maxLevel  MerkleTreeLevel
 	maxDegree uint32
-	hasher    ledger.Hasher
+	hashFunc  HashFunc
 }
 
-func newMerkleTree(maxDegree uint32, hasher ledger.Hasher) (*merkleTree, error) {
+func newMerkleTree(maxDegree uint32, hashFunc HashFunc) (*merkleTree, error) {
 	if maxDegree < 2 {
 		return nil, errors.Errorf("maxDegree [%d] should not be less than 2 in the merkle tree", maxDegree)
 	}
 	return &merkleTree{
 		make(map[MerkleTreeLevel][]Hash),
 		1,
 		maxDegree,
-		hasher,
+		hashFunc,
 	}, nil
 }
 
@@ -176,7 +174,7 @@ func (m *merkleTree) update(nextLeafLevelHash Hash) error {
 		if uint32(len(currentLevelHashes)) <= m.maxDegree {
 			return nil
 		}
-		nextLevelHash, err := computeCombinedHash(currentLevelHashes, m.hasher)
+		nextLevelHash, err := computeCombinedHash(currentLevelHashes, m.hashFunc)
 		if err != nil {
 			return err
 		}
@@ -208,7 +206,7 @@ func (m *merkleTree) done() error {
 		case 1:
 			h = currentLevelHashes[0]
 		default:
-			if h, err = computeCombinedHash(currentLevelHashes, m.hasher); err != nil {
+			if h, err = computeCombinedHash(currentLevelHashes, m.hashFunc); err != nil {
 				return err
 			}
 		}
@@ -221,7 +219,7 @@ func (m *merkleTree) done() error {
 	if uint32(len(finalHashes)) > m.maxDegree {
 		delete(m.tree, m.maxLevel)
 		m.maxLevel++
-		combinedHash, err := computeCombinedHash(finalHashes, m.hasher)
+		combinedHash, err := computeCombinedHash(finalHashes, m.hashFunc)
 		if err != nil {
 			return err
 		}
@@ -252,12 +250,12 @@ func (m *merkleTree) String() string {
 	return fmt.Sprintf("tree := %#v", m.tree)
 }
 
-func computeCombinedHash(hashes []Hash, hasher ledger.Hasher) (Hash, error) {
+func computeCombinedHash(hashes []Hash, hashFunc HashFunc) (Hash, error) {
 	combinedHash := []byte{}
 	for _, h := range hashes {
 		combinedHash = append(combinedHash, h...)
 	}
-	return hasher.Hash(combinedHash, hashOpts)
+	return hashFunc(combinedHash)
 }
 
 func hashesToBytes(hashes []Hash) [][]byte {

core/ledger/kvledger/txmgmt/rwsetutil/query_results_helper_test.go

@@ -1,17 +1,7 @@
 /*
-Copyright IBM Corp. 2016 All Rights Reserved.
+Copyright IBM Corp. All Rights Reserved.
 
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-		 http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
+SPDX-License-Identifier: Apache-2.0
 */
 
 package rwsetutil
@@ -23,15 +13,23 @@ import (
 
 	"github.com/golang/protobuf/proto"
 	"github.com/hyperledger/fabric-protos-go/ledger/rwset/kvrwset"
-	"github.com/hyperledger/fabric/bccsp/sw"
 	"github.com/hyperledger/fabric/core/ledger/internal/version"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+var (
+	testHashFunc = func(data []byte) ([]byte, error) {
+		h := sha256.New()
+		if _, err := h.Write(data); err != nil {
+			return nil, err
+		}
+		return h.Sum(nil), nil
+	}
 )
 
 func TestQueryResultHelper_NoResults(t *testing.T) {
-	cryptoProvider, err := sw.NewDefaultSecurityLevelWithKeystore(sw.NewDummyKeyStore())
-	assert.NoError(t, err)
-	helper, _ := NewRangeQueryResultsHelper(true, 3, cryptoProvider)
+	helper, _ := NewRangeQueryResultsHelper(true, 3, testHashFunc)
 	r, h, err := helper.Done()
 	assert.NoError(t, err)
 	assert.Nil(t, h)
@@ -193,9 +191,7 @@ func TestQueryResultHelper_Hash_FirstLevelSkipNeededInDone(t *testing.T) {
 }
 
 func buildTestResults(t *testing.T, enableHashing bool, maxDegree int, kvReads []*kvrwset.KVRead) ([]*kvrwset.KVRead, *kvrwset.QueryReadsMerkleSummary) {
-	cryptoProvider, err := sw.NewDefaultSecurityLevelWithKeystore(sw.NewDummyKeyStore())
-	assert.NoError(t, err)
-	helper, _ := NewRangeQueryResultsHelper(enableHashing, uint32(maxDegree), cryptoProvider)
+	helper, _ := NewRangeQueryResultsHelper(enableHashing, uint32(maxDegree), testHashFunc)
 	for _, kvRead := range kvReads {
 		helper.AddResult(kvRead)
 	}
@@ -217,17 +213,13 @@ func computeTestHashKVReads(t *testing.T, kvReads []*kvrwset.KVRead) Hash {
 	queryReads.KvReads = kvReads
 	b, err := proto.Marshal(queryReads)
 	assert.NoError(t, err)
-	hash := sha256.New()
-	_, err = hash.Write(b)
-	assert.NoError(t, err)
-	h := hash.Sum(nil)
+	h, err := testHashFunc(b)
+	require.NoError(t, err)
 	return h
 }
 
 func computeTestCombinedHash(t *testing.T, hashes ...Hash) Hash {
-	cryptoProvider, err := sw.NewDefaultSecurityLevelWithKeystore(sw.NewDummyKeyStore())
-	assert.NoError(t, err)
-	h, err := computeCombinedHash(hashes, cryptoProvider)
+	h, err := computeCombinedHash(hashes, testHashFunc)
 	assert.NoError(t, err)
 	return h
 }