Skip to content

Commit 458c521

Browse files
mariwademariwade
committed
FAB-1336 Add new ledger blockstorage index.
Add a new ledger blockstorage index for History that will map (blocknum,trannum) to the file storage location for this block transaction This index will be used for the API GetTransactionsForKey() for (chaincode1,key1). It will do a key range query on chaincode1~key1 to pick up all chaincode1~key1 records. Results will indicate the set of (blocknum,trannum) transactions that updated this key. Change-Id: I81da09e5526d7e2966634c78a03d34011d514442 Signed-off-by: Mari Wade <mariwade@us.ibm.com>
1 parent d18aa98 commit 458c521

File tree

9 files changed

+121
-28
lines changed

9 files changed

+121
-28
lines changed

‎core/ledger/blkstorage/blockstorage.go

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,9 +30,10 @@ type IndexableAttr string
3030

3131
// constants for indexable attributes
3232
const (
33-
IndexableAttrBlockNum = IndexableAttr("BlockNum")
34-
IndexableAttrBlockHash = IndexableAttr("BlockHash")
35-
IndexableAttrTxID = IndexableAttr("TxID")
33+
IndexableAttrBlockNum = IndexableAttr("BlockNum")
34+
IndexableAttrBlockHash = IndexableAttr("BlockHash")
35+
IndexableAttrTxID = IndexableAttr("TxID")
36+
IndexableAttrBlockNumTranNum = IndexableAttr("BlockNumTranNum")
3637
)
3738

3839
// IndexConfig - a configuration that includes a list of attributes that should be indexed

‎core/ledger/blkstorage/fsblkstorage/block_serialization.go

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,13 @@ import (
2727

2828
type serializedBlockInfo struct {
2929
blockHeader *common.BlockHeader
30-
txOffsets map[string]*locPointer
30+
txOffsets []*txindexInfo
31+
}
32+
33+
//The order of the transactions must be maintained for history
34+
type txindexInfo struct {
35+
txID string
36+
loc *locPointer
3137
}
3238

3339
func serializeBlock(block *common.Block) ([]byte, *serializedBlockInfo, error) {
@@ -94,8 +100,9 @@ func addHeaderBytes(blockHeader *common.BlockHeader, buf *proto.Buffer) error {
94100
return nil
95101
}
96102

97-
func addDataBytes(blockData *common.BlockData, buf *proto.Buffer) (map[string]*locPointer, error) {
98-
txOffsets := make(map[string]*locPointer)
103+
func addDataBytes(blockData *common.BlockData, buf *proto.Buffer) ([]*txindexInfo, error) {
104+
var txOffsets []*txindexInfo
105+
99106
if err := buf.EncodeVarint(uint64(len(blockData.Data))); err != nil {
100107
return nil, err
101108
}
@@ -108,7 +115,8 @@ func addDataBytes(blockData *common.BlockData, buf *proto.Buffer) (map[string]*l
108115
if err := buf.EncodeRawBytes(txEnvelopeBytes); err != nil {
109116
return nil, err
110117
}
111-
txOffsets[txid] = &locPointer{offset, len(buf.Bytes()) - offset}
118+
idxInfo := &txindexInfo{txid, &locPointer{offset, len(buf.Bytes()) - offset}}
119+
txOffsets = append(txOffsets, idxInfo)
112120
}
113121
return txOffsets, nil
114122
}
@@ -147,9 +155,9 @@ func extractHeader(buf *ledgerutil.Buffer) (*common.BlockHeader, error) {
147155
return header, nil
148156
}
149157

150-
func extractData(buf *ledgerutil.Buffer) (*common.BlockData, map[string]*locPointer, error) {
158+
func extractData(buf *ledgerutil.Buffer) (*common.BlockData, []*txindexInfo, error) {
151159
data := &common.BlockData{}
152-
txOffsets := make(map[string]*locPointer)
160+
var txOffsets []*txindexInfo
153161
var numItems uint64
154162
var err error
155163

@@ -167,7 +175,8 @@ func extractData(buf *ledgerutil.Buffer) (*common.BlockData, map[string]*locPoin
167175
return nil, nil, err
168176
}
169177
data.Data = append(data.Data, txEnvBytes)
170-
txOffsets[txid] = &locPointer{txOffset, buf.GetBytesConsumed() - txOffset}
178+
idxInfo := &txindexInfo{txid, &locPointer{txOffset, buf.GetBytesConsumed() - txOffset}}
179+
txOffsets = append(txOffsets, idxInfo)
171180
}
172181
return data, txOffsets, nil
173182
}

‎core/ledger/blkstorage/fsblkstorage/block_serialization_test.go

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -50,12 +50,16 @@ func TestSerializedBlockInfo(t *testing.T) {
5050
testutil.AssertNoError(t, err, "")
5151
testutil.AssertEquals(t, infoFromBB, info)
5252
testutil.AssertEquals(t, len(info.txOffsets), len(block.Data.Data))
53-
for _, txEnvBytes := range block.Data.Data {
53+
for txIndex, txEnvBytes := range block.Data.Data {
5454
txid, err := extractTxID(txEnvBytes)
5555
testutil.AssertNoError(t, err, "")
56-
offset, ok := info.txOffsets[txid]
57-
testutil.AssertEquals(t, ok, true)
58-
b := bb[offset.offset:]
56+
57+
indexInfo := info.txOffsets[txIndex]
58+
indexTxID := indexInfo.txID
59+
indexOffset := indexInfo.loc
60+
61+
testutil.AssertEquals(t, txid, indexTxID)
62+
b := bb[indexOffset.offset:]
5963
len, num := proto.DecodeVarint(b)
6064
txEnvBytesFromBB := b[num : num+int(len)]
6165
testutil.AssertEquals(t, txEnvBytesFromBB, txEnvBytes)

‎core/ledger/blkstorage/fsblkstorage/blockfile_mgr.go

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -304,7 +304,7 @@ func (mgr *blockfileMgr) addBlock(block *common.Block) error {
304304
blockFLP.offset = currentOffset
305305
// shift the txoffset because we prepend length of bytes before block bytes
306306
for _, txOffset := range txOffsets {
307-
txOffset.offset += len(blockBytesEncodedLen)
307+
txOffset.loc.offset += len(blockBytesEncodedLen)
308308
}
309309
//save the index in the database
310310
mgr.index.indexBlock(&blockIdxInfo{
@@ -363,7 +363,7 @@ func (mgr *blockfileMgr) syncIndex() error {
363363
return err
364364
}
365365
for _, offset := range info.txOffsets {
366-
offset.offset += int(blockPlacementInfo.blockBytesOffset)
366+
offset.loc.offset += int(blockPlacementInfo.blockBytesOffset)
367367
}
368368
//Update the blockIndexInfo with what was actually stored in file system
369369
blockIdxInfo := &blockIdxInfo{}
@@ -456,6 +456,15 @@ func (mgr *blockfileMgr) retrieveTransactionByID(txID string) (*pb.Transaction,
456456
return mgr.fetchTransaction(loc)
457457
}
458458

459+
func (mgr *blockfileMgr) retrieveTransactionForBlockNumTranNum(blockNum uint64, tranNum uint64) (*pb.Transaction, error) {
460+
logger.Debugf("retrieveTransactionForBlockNumTranNum() - blockNum = [%d], tranNum = [%d]", blockNum, tranNum)
461+
loc, err := mgr.index.getTXLocForBlockNumTranNum(blockNum, tranNum)
462+
if err != nil {
463+
return nil, err
464+
}
465+
return mgr.fetchTransaction(loc)
466+
}
467+
459468
func (mgr *blockfileMgr) fetchBlock(lp *fileLocPointer) (*common.Block, error) {
460469
blockBytes, err := mgr.fetchBlockBytes(lp)
461470
if err != nil {

‎core/ledger/blkstorage/fsblkstorage/blockindex.go

Lines changed: 51 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -27,10 +27,11 @@ import (
2727
)
2828

2929
const (
30-
blockNumIdxKeyPrefix = 'n'
31-
blockHashIdxKeyPrefix = 'h'
32-
txIDIdxKeyPrefix = 't'
33-
indexCheckpointKeyStr = "indexCheckpointKey"
30+
blockNumIdxKeyPrefix = 'n'
31+
blockHashIdxKeyPrefix = 'h'
32+
txIDIdxKeyPrefix = 't'
33+
blockNumTranNumIdxKeyPrefix = 'a'
34+
indexCheckpointKeyStr = "indexCheckpointKey"
3435
)
3536

3637
var indexCheckpointKey = []byte(indexCheckpointKeyStr)
@@ -41,13 +42,14 @@ type index interface {
4142
getBlockLocByHash(blockHash []byte) (*fileLocPointer, error)
4243
getBlockLocByBlockNum(blockNum uint64) (*fileLocPointer, error)
4344
getTxLoc(txID string) (*fileLocPointer, error)
45+
getTXLocForBlockNumTranNum(blockNum uint64, tranNum uint64) (*fileLocPointer, error)
4446
}
4547

4648
type blockIdxInfo struct {
4749
blockNum uint64
4850
blockHash []byte
4951
flp *fileLocPointer
50-
txOffsets map[string]*locPointer
52+
txOffsets []*txindexInfo
5153
}
5254

5355
type blockIndex struct {
@@ -89,25 +91,42 @@ func (index *blockIndex) indexBlock(blockIdxInfo *blockIdxInfo) error {
8991
return err
9092
}
9193

94+
//Index1
9295
if _, ok := index.indexItemsMap[blkstorage.IndexableAttrBlockHash]; ok {
9396
batch.Put(constructBlockHashKey(blockIdxInfo.blockHash), flpBytes)
9497
}
9598

99+
//Index2
96100
if _, ok := index.indexItemsMap[blkstorage.IndexableAttrBlockNum]; ok {
97101
batch.Put(constructBlockNumKey(blockIdxInfo.blockNum), flpBytes)
98102
}
99103

104+
//Index3 Used to find a transactin by it's transaction id
100105
if _, ok := index.indexItemsMap[blkstorage.IndexableAttrTxID]; ok {
101-
for txid, txoffset := range txOffsets {
102-
txFlp := newFileLocationPointer(flp.fileSuffixNum, flp.offset, txoffset)
103-
logger.Debugf("Adding txLoc [%s] for tx [%s] to index", txFlp, txid)
106+
for _, txoffset := range txOffsets {
107+
txFlp := newFileLocationPointer(flp.fileSuffixNum, flp.offset, txoffset.loc)
108+
logger.Debugf("Adding txLoc [%s] for tx ID: [%s] to index", txFlp, txoffset.txID)
104109
txFlpBytes, marshalErr := txFlp.marshal()
105110
if marshalErr != nil {
106111
return marshalErr
107112
}
108-
batch.Put(constructTxIDKey(txid), txFlpBytes)
113+
batch.Put(constructTxIDKey(txoffset.txID), txFlpBytes)
109114
}
110115
}
116+
117+
//Index4 - Store BlockNumTranNum will be used to query history data
118+
if _, ok := index.indexItemsMap[blkstorage.IndexableAttrBlockNumTranNum]; ok {
119+
for txIterator, txoffset := range txOffsets {
120+
txFlp := newFileLocationPointer(flp.fileSuffixNum, flp.offset, txoffset.loc)
121+
logger.Debugf("Adding txLoc [%s] for tx number:[%d] ID: [%s] to blockNumTranNum index", txFlp, txIterator+1, txoffset.txID)
122+
txFlpBytes, marshalErr := txFlp.marshal()
123+
if marshalErr != nil {
124+
return marshalErr
125+
}
126+
batch.Put(constructBlockNumTranNumKey(blockIdxInfo.blockNum, uint64(txIterator+1)), txFlpBytes)
127+
}
128+
}
129+
111130
batch.Put(indexCheckpointKey, encodeBlockNum(blockIdxInfo.blockNum))
112131
if err := index.db.WriteBatch(batch, false); err != nil {
113132
return err
@@ -163,6 +182,22 @@ func (index *blockIndex) getTxLoc(txID string) (*fileLocPointer, error) {
163182
return txFLP, nil
164183
}
165184

185+
func (index *blockIndex) getTXLocForBlockNumTranNum(blockNum uint64, tranNum uint64) (*fileLocPointer, error) {
186+
if _, ok := index.indexItemsMap[blkstorage.IndexableAttrBlockNumTranNum]; !ok {
187+
return nil, blkstorage.ErrAttrNotIndexed
188+
}
189+
b, err := index.db.Get(constructBlockNumTranNumKey(blockNum, tranNum))
190+
if err != nil {
191+
return nil, err
192+
}
193+
if b == nil {
194+
return nil, blkstorage.ErrNotFoundInIndex
195+
}
196+
txFLP := &fileLocPointer{}
197+
txFLP.unmarshal(b)
198+
return txFLP, nil
199+
}
200+
166201
func constructBlockNumKey(blockNum uint64) []byte {
167202
blkNumBytes := util.EncodeOrderPreservingVarUint64(blockNum)
168203
return append([]byte{blockNumIdxKeyPrefix}, blkNumBytes...)
@@ -176,6 +211,13 @@ func constructTxIDKey(txID string) []byte {
176211
return append([]byte{txIDIdxKeyPrefix}, []byte(txID)...)
177212
}
178213

214+
func constructBlockNumTranNumKey(blockNum uint64, txNum uint64) []byte {
215+
blkNumBytes := util.EncodeOrderPreservingVarUint64(blockNum)
216+
tranNumBytes := util.EncodeOrderPreservingVarUint64(txNum)
217+
key := append(blkNumBytes, tranNumBytes...)
218+
return append([]byte{blockNumTranNumIdxKeyPrefix}, key...)
219+
}
220+
179221
func constructTxID(blockNum uint64, txNum int) string {
180222
return fmt.Sprintf("%d:%d", blockNum, txNum)
181223
}

‎core/ledger/blkstorage/fsblkstorage/blockindex_test.go

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,9 @@ func (i *noopIndex) getBlockLocByBlockNum(blockNum uint64) (*fileLocPointer, err
4242
func (i *noopIndex) getTxLoc(txID string) (*fileLocPointer, error) {
4343
return nil, nil
4444
}
45+
func (i *noopIndex) getTXLocForBlockNumTranNum(blockNum uint64, tranNum uint64) (*fileLocPointer, error) {
46+
return nil, nil
47+
}
4548

4649
func TestBlockIndexSync(t *testing.T) {
4750
testBlockIndexSync(t, 10, 5, false)
@@ -103,7 +106,9 @@ func TestBlockIndexSelectiveIndexing(t *testing.T) {
103106
testBlockIndexSelectiveIndexing(t, []blkstorage.IndexableAttr{blkstorage.IndexableAttrBlockHash})
104107
testBlockIndexSelectiveIndexing(t, []blkstorage.IndexableAttr{blkstorage.IndexableAttrBlockNum})
105108
testBlockIndexSelectiveIndexing(t, []blkstorage.IndexableAttr{blkstorage.IndexableAttrTxID})
109+
testBlockIndexSelectiveIndexing(t, []blkstorage.IndexableAttr{blkstorage.IndexableAttrBlockNumTranNum})
106110
testBlockIndexSelectiveIndexing(t, []blkstorage.IndexableAttr{blkstorage.IndexableAttrBlockHash, blkstorage.IndexableAttrBlockNum})
111+
testBlockIndexSelectiveIndexing(t, []blkstorage.IndexableAttr{blkstorage.IndexableAttrTxID, blkstorage.IndexableAttrBlockNumTranNum})
107112
}
108113

109114
func testBlockIndexSelectiveIndexing(t *testing.T, indexItems []blkstorage.IndexableAttr) {
@@ -149,4 +154,15 @@ func testBlockIndexSelectiveIndexing(t *testing.T, indexItems []blkstorage.Index
149154
} else {
150155
testutil.AssertSame(t, err, blkstorage.ErrAttrNotIndexed)
151156
}
157+
158+
//test 'retrieveTrasnactionsByBlockNumTranNum
159+
tx2, err := blockfileMgr.retrieveTransactionForBlockNumTranNum(1, 1)
160+
if testutil.Contains(indexItems, blkstorage.IndexableAttrBlockNumTranNum) {
161+
testutil.AssertNoError(t, err, "Error while retrieving tx by blockNum and tranNum")
162+
txOrig2, err2 := extractTransaction(blocks[0].Data.Data[0])
163+
testutil.AssertNoError(t, err2, "")
164+
testutil.AssertEquals(t, tx2, txOrig2)
165+
} else {
166+
testutil.AssertSame(t, err, blkstorage.ErrAttrNotIndexed)
167+
}
152168
}

‎core/ledger/blkstorage/fsblkstorage/pkg_test.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ func newTestEnv(t testing.TB) *testEnv {
3939
blkstorage.IndexableAttrBlockHash,
4040
blkstorage.IndexableAttrBlockNum,
4141
blkstorage.IndexableAttrTxID,
42+
blkstorage.IndexableAttrBlockNumTranNum,
4243
}
4344
os.RemoveAll(conf.dbPath)
4445
os.RemoveAll(conf.blockfilesDir)

‎core/ledger/kvledger/kv_ledger.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,7 @@ func NewKVLedger(conf *Conf) (*KVLedger, error) {
7474
blkstorage.IndexableAttrBlockHash,
7575
blkstorage.IndexableAttrBlockNum,
7676
blkstorage.IndexableAttrTxID,
77+
blkstorage.IndexableAttrBlockNumTranNum,
7778
}
7879
indexConfig := &blkstorage.IndexConfig{AttrsToIndex: attrsToIndex}
7980
blockStorageConf := fsblkstorage.NewConf(conf.blockStorageDir, conf.maxBlockfileSize)

‎core/ledger/kvledger/kv_ledger_test.go

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -179,7 +179,7 @@ func TestLedgerWithCouchDbEnabledWithBinaryAndJSONData(t *testing.T) {
179179
simulator, _ := ledger.NewTxSimulator()
180180
simulator.SetState("ns1", "key4", []byte("value1"))
181181
simulator.SetState("ns1", "key5", []byte("value2"))
182-
simulator.SetState("ns1", "key6", []byte("{\"shipmentID\":\"161003PKC7300\",\"customsInvoice\":{\"methodOfTransport\":\"GROUND\",\"invoiceNumber\":\"00091624\"},\"weightUnitOfMeasure\":\"KGM\",\"volumeUnitOfMeasure\": \"CO\",\"dimensionUnitOfMeasure\":\"CM\",\"currency\":\"USD\"}"))
182+
simulator.SetState("ns1", "key6", []byte("{\"shipmentID\":\"161003PKC7300\",\"customsInvoice\":{\"methodOfTransport\":\"GROUND\",\"invoiceNumber\":\"00091622\"},\"weightUnitOfMeasure\":\"KGM\",\"volumeUnitOfMeasure\": \"CO\",\"dimensionUnitOfMeasure\":\"CM\",\"currency\":\"USD\"}"))
183183
simulator.SetState("ns1", "key7", []byte("{\"shipmentID\":\"161003PKC7600\",\"customsInvoice\":{\"methodOfTransport\":\"AIR MAYBE\",\"invoiceNumber\":\"00091624\"},\"weightUnitOfMeasure\":\"KGM\",\"volumeUnitOfMeasure\": \"CO\",\"dimensionUnitOfMeasure\":\"CM\",\"currency\":\"USD\"}"))
184184
simulator.Done()
185185
simRes, _ := simulator.GetTxSimulationResults()
@@ -195,6 +195,7 @@ func TestLedgerWithCouchDbEnabledWithBinaryAndJSONData(t *testing.T) {
195195
Height: 1, CurrentBlockHash: block1Hash, PreviousBlockHash: []byte{}})
196196

197197
//Note key 4 and 6 are updates but key 7 is new. I.E. should see history for key 4 and 6 if history is enabled
198+
simulationResults := [][]byte{}
198199
simulator, _ = ledger.NewTxSimulator()
199200
simulator.SetState("ns1", "key4", []byte("value3"))
200201
simulator.SetState("ns1", "key5", []byte("{\"shipmentID\":\"161003PKC7500\",\"customsInvoice\":{\"methodOfTransport\":\"AIR FREIGHT\",\"invoiceNumber\":\"00091623\"},\"weightUnitOfMeasure\":\"KGM\",\"volumeUnitOfMeasure\": \"CO\",\"dimensionUnitOfMeasure\":\"CM\",\"currency\":\"USD\"}"))
@@ -203,7 +204,16 @@ func TestLedgerWithCouchDbEnabledWithBinaryAndJSONData(t *testing.T) {
203204
simulator.SetState("ns1", "key8", []byte("{\"shipmentID\":\"161003PKC7700\",\"customsInvoice\":{\"methodOfTransport\":\"SHIP\",\"invoiceNumber\":\"00091625\"},\"weightUnitOfMeasure\":\"KGM\",\"volumeUnitOfMeasure\": \"CO\",\"dimensionUnitOfMeasure\":\"CM\",\"currency\":\"USD\"}"))
204205
simulator.Done()
205206
simRes, _ = simulator.GetTxSimulationResults()
206-
block2 := bg.NextBlock([][]byte{simRes}, false)
207+
simulationResults = append(simulationResults, simRes)
208+
//add a 2nd transaction
209+
simulator2, _ := ledger.NewTxSimulator()
210+
simulator2.SetState("ns1", "key9", []byte("value5"))
211+
simulator2.SetState("ns1", "key10", []byte("{\"shipmentID\":\"261003PKC8000\",\"customsInvoice\":{\"methodOfTransport\":\"DONKEY\",\"invoiceNumber\":\"00091626\"},\"weightUnitOfMeasure\":\"KGM\",\"volumeUnitOfMeasure\": \"CO\",\"dimensionUnitOfMeasure\":\"CM\",\"currency\":\"USD\"}"))
212+
simulator2.Done()
213+
simRes2, _ := simulator2.GetTxSimulationResults()
214+
simulationResults = append(simulationResults, simRes2)
215+
216+
block2 := bg.NextBlock(simulationResults, false)
207217
ledger.RemoveInvalidTransactionsAndPrepare(block2)
208218
ledger.Commit()
209219

@@ -225,6 +235,6 @@ func TestLedgerWithCouchDbEnabledWithBinaryAndJSONData(t *testing.T) {
225235
testutil.AssertEquals(t, b2, block2)
226236

227237
if ledgerconfig.IsHistoryDBEnabled() == true {
228-
//TODO history specific test
238+
//TODO history specific test once the query api's are in and we can validate content
229239
}
230240
}

0 commit comments

Comments
 (0)