From a19f67ea52d7450f48a00dfe74a461a0a8e8288c Mon Sep 17 00:00:00 2001 From: eunsoopark Date: Tue, 19 Nov 2019 09:22:42 +0900 Subject: [PATCH] IS-873: send last commit block height with VERSION, READY message - refactor DBInfoData with DBInfoDataV2 --- cmd/iissdata/cli_read.go | 62 ++++++++++++++++++++++++++++++-- core/context.go | 72 +++++++++++++++++++++++--------------- core/context_test.go | 72 +++++++++++++++++++++++--------------- core/db_claim.go | 8 ++++- core/db_claim_test.go | 7 ++++ core/db_iissdata.go | 56 ----------------------------- core/db_iissdata_test.go | 22 ++++++++++++ core/db_management.go | 60 +++++++++++++++++++++++++------ core/db_management_test.go | 65 ++++++++++++++++++++++++---------- core/msg.go | 14 +++++--- core/msg_calculate.go | 34 +++++++++--------- core/msg_calculate_test.go | 26 +++++++------- core/msg_claim.go | 3 ++ core/msg_debug.go | 2 +- core/msg_rollback.go | 12 ++++--- core/msg_rollback_test.go | 6 ++-- 16 files changed, 332 insertions(+), 189 deletions(-) diff --git a/cmd/iissdata/cli_read.go b/cmd/iissdata/cli_read.go index b6244f0..6194842 100644 --- a/cmd/iissdata/cli_read.go +++ b/cmd/iissdata/cli_read.go @@ -3,6 +3,8 @@ package main import ( "encoding/hex" "fmt" + "github.com/icon-project/rewardcalculator/common" + "log" "os" "path/filepath" @@ -24,8 +26,8 @@ func (cli *CLI) read(dbDir string, dbName string) { iissDB := core.OpenIISSData(path) core.LoadIISSData(iissDB) - core.ReadIISSBP(iissDB) - core.ReadIISSTX(iissDB) + ReadIISSBP(iissDB) + ReadIISSTX(iissDB) //checkIISSTX(iissDB, lossDB) @@ -34,6 +36,62 @@ func (cli *CLI) read(dbDir string, dbName string) { iissDB.Close() } +func ReadIISSBP(iissDB db.Database) { + var bpInfo core.IISSBlockProduceInfo + + iter, _ := iissDB.GetIterator() + prefix := util.BytesPrefix([]byte(db.PrefixIISSBPInfo)) + iter.New(prefix.Start, prefix.Limit) + var entries, startBH, miss uint64 + for entries = 0; iter.Next(); entries++ { + err := bpInfo.SetBytes(iter.Value()) + if err != nil { + log.Printf("Failed to load IISS Block Produce information.") + continue + } + bpInfo.BlockHeight = common.BytesToUint64(iter.Key()[len(db.PrefixIISSBPInfo):]) + if entries == 0 { + startBH = bpInfo.BlockHeight + } + if startBH + entries + miss != bpInfo.BlockHeight { + fmt.Printf("Miss BP block height %d\n", startBH + entries + miss) + miss++ + for ; startBH + entries + miss != bpInfo.BlockHeight; miss++ { + fmt.Printf("Miss BP block height %d\n", startBH + entries + miss) + } + } + } + log.Printf(">> BP total count %d, miss %d", entries, miss) + iter.Release() + err := iter.Error() + if err != nil { + log.Printf("There is error while read IISS BP iteration. %+v", err) + } +} + +func ReadIISSTX(iissDB db.Database) { + var tx core.IISSTX + + iter, _ := iissDB.GetIterator() + prefix := util.BytesPrefix([]byte(db.PrefixIISSTX)) + iter.New(prefix.Start, prefix.Limit) + entries := 0 + for entries = 0; iter.Next(); entries++ { + err := tx.SetBytes(iter.Value()) + if err != nil { + log.Printf("Failed to load IISS TX data") + continue + } + //log.Printf("[IISSTX] TX : %s", tx.String()) + } + log.Printf(">> TX total count %d", entries) + iter.Release() + err := iter.Error() + if err != nil { + log.Printf("There is error while read IISS TX iteration. %+v", err) + } +} + func checkIISSTX(iissDB db.Database, result db.Database) { var tx core.IISSTX diff --git a/core/context.go b/core/context.go index 2ef3b4e..66e6a5e 100644 --- a/core/context.go +++ b/core/context.go @@ -125,7 +125,6 @@ func (idb *IScoreDB) getCalculateResultDB() db.Database { return idb.calcResult } - func (idb *IScoreDB) resetAccountDB(blockHeight uint64) error { idb.accountLock.Lock() defer idb.accountLock.Unlock() @@ -182,43 +181,60 @@ func (idb *IScoreDB) resetAccountDB(blockHeight uint64) error { return nil } -func (idb *IScoreDB) setCalculateBlockHeight(blockHeight uint64) { - idb.info.CalcBlockHeight = blockHeight +func (idb *IScoreDB) setCalculatingBH(blockHeight uint64) { + idb.info.Calculating = blockHeight idb.writeToDB() } -func (idb *IScoreDB) resetCalculateBlockHeight() { - idb.info.CalcBlockHeight = idb.info.BlockHeight +func (idb *IScoreDB) resetCalculatingBH() { + idb.info.Calculating = idb.info.CalcDone idb.writeToDB() } func (idb *IScoreDB) isCalculating() bool { - return idb.info.CalcBlockHeight > idb.info.BlockHeight + return idb.getCalculatingBH() > idb.getCalcDoneBH() } -func (idb *IScoreDB) setBlockInfo(blockHeight uint64, blockHash []byte) { - tempBlockHash := make([]byte, BlockHashSize) - copy(tempBlockHash, blockHash) +func (idb *IScoreDB) setCurrentBlockInfo(blockHeight uint64, blockHash []byte) { + idb.info.Current.set(blockHeight, blockHash) - // backup to prev Info. - if idb.info.PrevBlockHeight < blockHeight { - idb.info.PrevBlockHeight = idb.info.BlockHeight - copy(idb.info.PrevBlockHash, idb.info.BlockHash) - } + idb.writeToDB() +} + +func (idb *IScoreDB) setCalcDoneBH(blockHeight uint64) { + idb.info.PrevCalcDone = idb.info.CalcDone + idb.info.CalcDone = blockHeight + + idb.writeToDB() +} + +func (idb *IScoreDB) getCurrentBlockInfo() *BlockInfo { + return &idb.info.Current +} - // set current Info. - idb.info.BlockHeight = blockHeight - copy(idb.info.BlockHash, tempBlockHash) +func (idb *IScoreDB) getCalcDoneBH() uint64 { + return idb.info.CalcDone +} + +func (idb *IScoreDB) getPrevCalcDoneBH() uint64 { + return idb.info.PrevCalcDone +} + +func (idb *IScoreDB) getCalculatingBH() uint64 { + return idb.info.Calculating +} + +func (idb *IScoreDB) rollbackCurrentBlockInfo(blockHeight uint64, blockHash []byte) { + idb.setCurrentBlockInfo(blockHeight, blockHash) idb.writeToDB() } -func (idb *IScoreDB) rollbackBlockInfo() { - idb.info.BlockHeight = idb.info.PrevBlockHeight - copy(idb.info.BlockHash, idb.info.PrevBlockHash) - idb.setCalculateBlockHeight(idb.info.BlockHeight) +func (idb *IScoreDB) rollbackAccountDBBlockInfo() { + idb.info.CalcDone = idb.info.PrevCalcDone + idb.info.Calculating = idb.info.PrevCalcDone idb.writeToDB() } @@ -286,11 +302,11 @@ func (idb *IScoreDB) rollbackAccountDB(blockHeight uint64) error { // toggle query DB switch idb.toggleAccountDB() - // Rollback block height and block hash - idb.rollbackBlockInfo() - // delete calculation result - DeleteCalculationResult(idb.getCalculateResultDB(), idb.info.CalcBlockHeight) + DeleteCalculationResult(idb.getCalculateResultDB(), idb.getCalcDoneBH()) + + // Rollback block height and block hash + idb.rollbackAccountDBBlockInfo() log.Printf("End rollblack account DB to %d", blockHeight) return nil @@ -342,7 +358,7 @@ func (ctx *Context) UpdateGovernanceVariable(gvList []*IISSGovernanceVariable) { deleteOld := false deleteIndex := -1 for i := gvLen - 1; i >= 0 ; i-- { - if ctx.GV[i].BlockHeight < ctx.DB.info.BlockHeight { + if ctx.GV[i].BlockHeight < ctx.DB.getCalcDoneBH() { if deleteOld { // delete from management DB bucket.Delete(ctx.GV[i].ID()) @@ -377,7 +393,7 @@ func (ctx *Context) UpdatePRep(prepList []*PRep) { deleteOld := false deleteIndex := -1 for i := prepLen - 1; i >= 0 ; i-- { - if ctx.PRep[i].BlockHeight < ctx.DB.info.BlockHeight { + if ctx.PRep[i].BlockHeight < ctx.DB.getCalcDoneBH() { if deleteOld { // delete from management DB bucket.Delete(ctx.PRep[i].ID()) @@ -494,7 +510,7 @@ func NewContext(dbPath string, dbType string, dbName string, dbCount int) (*Cont } // read Governance variable - ctx.GV, err = LoadGovernanceVariable(mngDB, ctx.DB.info.BlockHeight) + ctx.GV, err = LoadGovernanceVariable(mngDB, ctx.DB.getCalcDoneBH()) if err != nil { log.Printf("Failed to load GV structure\n") return nil, err diff --git a/core/context_test.go b/core/context_test.go index 62b58ec..c0ac81c 100644 --- a/core/context_test.go +++ b/core/context_test.go @@ -41,7 +41,10 @@ func TestContext_NewContext(t *testing.T) { assert.NotNil(t, ctx.DB) assert.NotNil(t, ctx.DB.info) assert.Equal(t, dbCount, ctx.DB.info.DBCount) - assert.Equal(t, uint64(0), ctx.DB.info.BlockHeight) + assert.True(t, ctx.DB.getCurrentBlockInfo().checkValue(uint64(0), zeroBlockHash)) + assert.Equal(t, uint64(0), ctx.DB.getCalcDoneBH()) + assert.Equal(t, uint64(0), ctx.DB.getPrevCalcDoneBH()) + assert.Equal(t, uint64(0), ctx.DB.getCalculatingBH()) assert.False(t, ctx.DB.info.QueryDBIsZero) assert.Equal(t, dbCount, len(ctx.DB.Account0)) assert.Equal(t, dbCount, len(ctx.DB.Account1)) @@ -64,7 +67,9 @@ func TestContext_UpdateGovernanceVariable(t *testing.T) { bucket, _ := ctx.DB.management.GetBucket(db.PrefixGovernanceVariable) // Set Block height - ctx.DB.info.BlockHeight = ctxBlockHeight + blockHash := make([]byte, BlockHashSize) + copy(blockHash, []byte(string(ctxBlockHeight))) + ctx.DB.setCalcDoneBH(ctxBlockHeight) // Insert initial GV gv := new(GovernanceVariable) @@ -135,7 +140,9 @@ func TestContext_UpdatePRep(t *testing.T) { bucket, _ := ctx.DB.management.GetBucket(db.PrefixPRep) // Set Block height - ctx.DB.info.BlockHeight = ctxBlockHeight + blockHash := make([]byte, BlockHashSize) + copy(blockHash, []byte(string(ctxBlockHeight))) + ctx.DB.setCalcDoneBH(ctxBlockHeight) // Insert initial PRep pRep := new(PRep) @@ -437,7 +444,7 @@ func TestContext_ResetAccountDB(t *testing.T) { backupDB.Close() } -func TestContext_rollbackBlockInfo(t *testing.T) { +func TestContext_CurrentBlockInfo(t *testing.T) { ctx := initTest(1) defer finalizeTest(ctx) @@ -451,16 +458,32 @@ func TestContext_rollbackBlockInfo(t *testing.T) { blockHash2 := make([]byte, BlockHashSize) copy(blockHash2, []byte(string(blockHeight2))) - ctx.DB.setBlockInfo(blockHeight1, blockHash1) - ctx.DB.setBlockInfo(blockHeight2, blockHash2) + ctx.DB.setCurrentBlockInfo(blockHeight2, blockHash2) + assert.True(t, ctx.DB.getCurrentBlockInfo().checkValue(blockHeight2, blockHash2)) - ctx.DB.rollbackBlockInfo() + ctx.DB.rollbackCurrentBlockInfo(blockHeight1, blockHash1) + assert.True(t, ctx.DB.getCurrentBlockInfo().checkValue(blockHeight1, blockHash1)) +} + +func TestContext_AccountDBBlockHeight(t *testing.T) { + ctx := initTest(1) + defer finalizeTest(ctx) + + const ( + blockHeight1 uint64 = 100 + blockHeight2 uint64 = 200 + ) + + ctx.DB.setCalcDoneBH(blockHeight1) + ctx.DB.setCalcDoneBH(blockHeight2) + assert.Equal(t, blockHeight2, ctx.DB.getCalcDoneBH()) + assert.Equal(t, blockHeight1, ctx.DB.getPrevCalcDoneBH()) + + ctx.DB.rollbackAccountDBBlockInfo() - assert.Equal(t, blockHeight1, ctx.DB.info.BlockHeight) - assert.Equal(t, blockHash1, ctx.DB.info.BlockHash) - assert.Equal(t, blockHeight1, ctx.DB.info.PrevBlockHeight) - assert.Equal(t, blockHash1, ctx.DB.info.PrevBlockHash) - assert.Equal(t, blockHeight1, ctx.DB.info.CalcBlockHeight) + assert.Equal(t, blockHeight1, ctx.DB.getCalcDoneBH()) + assert.Equal(t, blockHeight1, ctx.DB.getPrevCalcDoneBH()) + assert.Equal(t, blockHeight1, ctx.DB.getCalculatingBH()) } func TestContext_RollbackAccountDB(t *testing.T) { @@ -482,11 +505,9 @@ func TestContext_RollbackAccountDB(t *testing.T) { // emulate calculation process prevBlockHeight := uint64(5) - prevBlockHash := testBlockHash0 - ctx.DB.setBlockInfo(prevBlockHeight, prevBlockHash) + ctx.DB.setCalcDoneBH(prevBlockHeight) blockHeight := uint64(10) - blockHash := testBlockHash - ctx.DB.setCalculateBlockHeight(blockHeight) + ctx.DB.setCalculatingBH(blockHeight) ctx.DB.writeToDB() ctx.DB.toggleAccountDB() @@ -498,12 +519,10 @@ func TestContext_RollbackAccountDB(t *testing.T) { err = ctx.DB.resetAccountDB(blockHeight) assert.Nil(t, err) WriteCalculationResult(crDB, blockHeight, nil, nil) - ctx.DB.setBlockInfo(blockHeight, blockHash) + ctx.DB.setCalcDoneBH(blockHeight) ctx.DB.writeToDB() - assert.Equal(t, prevBlockHeight, ctx.DB.info.PrevBlockHeight) - assert.Equal(t, prevBlockHash, ctx.DB.info.PrevBlockHash) - assert.Equal(t, blockHeight, ctx.DB.info.BlockHeight) - assert.Equal(t, blockHash, ctx.DB.info.BlockHash) + assert.Equal(t, prevBlockHeight, ctx.DB.getPrevCalcDoneBH()) + assert.Equal(t, blockHeight, ctx.DB.getCalcDoneBH()) // read from query DB qDB = ctx.DB.getQueryDB(ia.Address) @@ -511,7 +530,7 @@ func TestContext_RollbackAccountDB(t *testing.T) { bs, _ := bucket.Get(ia.ID()) assert.Nil(t, bs) - // no need to Rollback with blockHeigh >= ctx.DB.Info.CalcBlockHeight + // no need to Rollback with blockHeight >= ctx.DB.Info.CalcBlockHeight err = ctx.DB.rollbackAccountDB(blockHeight) // backup account DB remains @@ -521,8 +540,7 @@ func TestContext_RollbackAccountDB(t *testing.T) { assert.True(t, stat.IsDir()) // check block height and block hash - assert.Equal(t, blockHeight, ctx.DB.info.BlockHeight) - assert.Equal(t, blockHash, ctx.DB.info.BlockHash) + assert.Equal(t, blockHeight, ctx.DB.getCalcDoneBH()) // valid Rollback err = ctx.DB.rollbackAccountDB(0) @@ -539,10 +557,8 @@ func TestContext_RollbackAccountDB(t *testing.T) { assert.Nil(t, bs) // check Rollback block height and block hash - assert.Equal(t, prevBlockHeight, ctx.DB.info.PrevBlockHeight) - assert.Equal(t, prevBlockHash, ctx.DB.info.PrevBlockHash) - assert.Equal(t, prevBlockHeight, ctx.DB.info.BlockHeight) - assert.Equal(t, prevBlockHash, ctx.DB.info.BlockHash) + assert.Equal(t, prevBlockHeight, ctx.DB.getCalcDoneBH()) + assert.Equal(t, prevBlockHeight, ctx.DB.getPrevCalcDoneBH()) // read from query DB qDB = ctx.DB.getQueryDB(ia.Address) diff --git a/core/db_claim.go b/core/db_claim.go index b1fa3f0..ba8274e 100644 --- a/core/db_claim.go +++ b/core/db_claim.go @@ -432,12 +432,16 @@ func checkClaimDBRollback(cbInfo *ClaimBackupInfo, rollback uint64) (bool, error return true, nil } -func rollbackClaimDB(cDB db.Database, cbDB db.Database, to uint64) error { +func rollbackClaimDB(ctx *Context, to uint64, blockHash []byte) error { log.Printf("Start Rollback claim DB to %d", to) + idb := ctx.DB + cDB := idb.getClaimDB() + cbDB := idb.getClaimBackupDB() bucket, err := cbDB.GetBucket(db.PrefixManagement) if err != nil { return err } + var cbInfo ClaimBackupInfo bs, _ := bucket.Get(cbInfo.ID()) cbInfo.SetBytes(bs) @@ -465,6 +469,8 @@ func rollbackClaimDB(cDB db.Database, cbDB db.Database, to uint64) error { cbInfo.LastBlockHeight = to bucket.Set(cbInfo.ID(), cbInfo.Bytes()) + idb.rollbackCurrentBlockInfo(to, blockHash) + log.Printf("End Rollback claim DB from %d to %d", from, to) return nil } diff --git a/core/db_claim_test.go b/core/db_claim_test.go index 050a2af..c92250d 100644 --- a/core/db_claim_test.go +++ b/core/db_claim_test.go @@ -79,6 +79,13 @@ var testBlockHash = []byte{ 0x12, 0x34, 0x56, 0x78, 0x90, 0xab, 0xcd, 0xef, } +var zeroBlockHash = []byte{ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +} + func makePreCommit() *PreCommit { claim := makeClaim() preCommit := newPreCommit(claimBlockHeight, testBlockHash, claim.Address) diff --git a/core/db_iissdata.go b/core/db_iissdata.go index d20c769..f16bcf0 100644 --- a/core/db_iissdata.go +++ b/core/db_iissdata.go @@ -203,39 +203,6 @@ func (bp *IISSBlockProduceInfo) SetBytes(bs []byte) error { return nil } -func ReadIISSBP(iissDB db.Database) { - var bpInfo IISSBlockProduceInfo - - iter, _ := iissDB.GetIterator() - prefix := util.BytesPrefix([]byte(db.PrefixIISSBPInfo)) - iter.New(prefix.Start, prefix.Limit) - var entries, startBH, miss uint64 - for entries = 0; iter.Next(); entries++ { - err := bpInfo.SetBytes(iter.Value()) - if err != nil { - log.Printf("Failed to load IISS Block Produce information.") - continue - } - bpInfo.BlockHeight = common.BytesToUint64(iter.Key()[len(db.PrefixIISSBPInfo):]) - if entries == 0 { - startBH = bpInfo.BlockHeight - } - if startBH + entries + miss != bpInfo.BlockHeight { - fmt.Printf("Miss BP block height %d\n", startBH + entries + miss) - miss++ - for ; startBH + entries + miss != bpInfo.BlockHeight; miss++ { - fmt.Printf("Miss BP block height %d\n", startBH + entries + miss) - } - } - } - log.Printf(">> BP total count %d, miss %d", entries, miss) - iter.Release() - err := iter.Error() - if err != nil { - log.Printf("There is error while read IISS BP iteration. %+v", err) - } -} - const ( TXDataTypeDelegate = 0 TXDataTypePrepReg = 1 @@ -288,29 +255,6 @@ func (tx *IISSTX) SetBytes(bs []byte) error { return nil } -func ReadIISSTX(iissDB db.Database) { - var tx IISSTX - - iter, _ := iissDB.GetIterator() - prefix := util.BytesPrefix([]byte(db.PrefixIISSTX)) - iter.New(prefix.Start, prefix.Limit) - entries := 0 - for entries = 0; iter.Next(); entries++ { - err := tx.SetBytes(iter.Value()) - if err != nil { - log.Printf("Failed to load IISS TX data") - continue - } - //log.Printf("[IISSTX] TX : %s", tx.String()) - } - log.Printf(">> TX total count %d", entries) - iter.Release() - err := iter.Error() - if err != nil { - log.Printf("There is error while read IISS TX iteration. %+v", err) - } -} - func OpenIISSData(path string) db.Database { dbPath := filepath.Clean(path) dbDir, dbName := filepath.Split(dbPath) diff --git a/core/db_iissdata_test.go b/core/db_iissdata_test.go index 7dea729..c58217b 100644 --- a/core/db_iissdata_test.go +++ b/core/db_iissdata_test.go @@ -2,6 +2,7 @@ package core import ( "os" + "path/filepath" "testing" "github.com/icon-project/rewardcalculator/common" @@ -410,3 +411,24 @@ func TestDBIISS_LoadIISSData(t *testing.T) { assert.NotNil(t, pRepListNew) assert.Equal(t, 0, len(pRepListNew)) } + +func TestDBIISS_manageIISSData(t *testing.T) { + rootPath, _ := filepath.Abs("./iissdata_test") + os.MkdirAll(rootPath, os.ModePerm) + iissPath := filepath.Join(rootPath, "current") + os.MkdirAll(iissPath, os.ModePerm) + finishPath := filepath.Join(rootPath, "finish_iiss") + os.MkdirAll(finishPath, os.ModePerm) + + cleanupIISSData(iissPath) + + _, err := os.Stat(iissPath) + assert.True(t, os.IsNotExist(err)) + _, err = os.Stat(finishPath) + assert.True(t, os.IsNotExist(err)) + backupPath := filepath.Join(rootPath, "finish_current") + f, err := os.Stat(backupPath) + assert.True(t, f.IsDir()) + + os.RemoveAll(rootPath) +} \ No newline at end of file diff --git a/core/db_management.go b/core/db_management.go index ae7c39e..f2ab9bd 100644 --- a/core/db_management.go +++ b/core/db_management.go @@ -1,6 +1,7 @@ package core import ( + "bytes" "encoding/json" "log" "math/big" @@ -21,16 +22,46 @@ const ( NumSubPRep uint64 = 78 ) -type DBInfoData struct { +type DBInfoDataV1 struct { DBCount int - BlockHeight uint64 // finish to calculate to thie block height + BlockHeight uint64 // finish to calculate to this block height QueryDBIsZero bool + BlockHash []byte CalcBlockHeight uint64 // try to calculate to this block height PrevBlockHeight uint64 // calculated to this block height in previous calculation PrevBlockHash []byte } +type BlockInfo struct { + BlockHeight uint64 + BlockHash [BlockHashSize]byte +} + +func (bi *BlockInfo) set(blockHeight uint64, blockHash []byte) { + bi.BlockHeight = blockHeight + copy(bi.BlockHash[:], blockHash) +} + +func (bi *BlockInfo) checkValue(blockHeight uint64, blockHash []byte) bool { + return bi.BlockHeight == blockHeight && bytes.Compare(bi.BlockHash[:], blockHash) == 0 +} + +func (bi *BlockInfo) equal(dst *BlockInfo) bool { + return bi.BlockHeight == dst.BlockHeight && bytes.Compare(bi.BlockHash[:], dst.BlockHash[:]) == 0 +} + +type DBInfoDataV2 struct { + DBCount int + QueryDBIsZero bool + Current BlockInfo // Latest COMMIT_BLOCK block height and hash + CalcDone uint64 // Latest CALCULATE_DONE block height + PrevCalcDone uint64 // Previous CALCULATE_DONE block height + Calculating uint64 // Latest CALCULATE block height +} + +type DBInfoData DBInfoDataV2 + type DBInfo struct { DBRoot string DBType string @@ -61,9 +92,24 @@ func (dbi *DBInfo) String() string { func (dbi *DBInfo) SetBytes(bs []byte) error { _, err := codec.UnmarshalFromBytes(bs, &dbi.DBInfoData) + if err != nil { + // handle backward compatibility + return dbi.backwardCompatibility(bs) + } + return nil +} + +func (dbi *DBInfo) backwardCompatibility(bs []byte) error { + var v1 DBInfoDataV1 + _, err := codec.UnmarshalFromBytes(bs, &v1) if err != nil { return err } + + dbi.DBCount = v1.DBCount + dbi.CalcDone = v1.BlockHeight + dbi.QueryDBIsZero = v1.QueryDBIsZero + return nil } @@ -91,12 +137,6 @@ func NewDBInfo(mngDB db.Database, dbPath string, dbType string, dbName string, d dbInfo.DBRoot = filepath.Join(dbPath, dbName) dbInfo.DBType = dbType - if dbInfo.BlockHash == nil || len(dbInfo.BlockHash) != BlockHashSize { - dbInfo.BlockHash = make([]byte, BlockHashSize) - } - if dbInfo.PrevBlockHash == nil || len(dbInfo.PrevBlockHash) != BlockHashSize { - dbInfo.PrevBlockHash = make([]byte, BlockHashSize) - } // Write to management DB if writeToDB { @@ -171,7 +211,7 @@ func (gv *GovernanceVariable) setReward() { gv.PRepReward.Mul(&gv.PRepReward.Int, BigIntIScoreMultiplier) } -func LoadGovernanceVariable(dbi db.Database, workingBH uint64) ([]*GovernanceVariable, error) { +func LoadGovernanceVariable(dbi db.Database, calcDoneBH uint64) ([]*GovernanceVariable, error) { gvList := make([]*GovernanceVariable, 0) iter, err := dbi.GetIterator() @@ -190,7 +230,7 @@ func LoadGovernanceVariable(dbi db.Database, workingBH uint64) ([]*GovernanceVar gv.SetBytes(iter.Value()) gv.BlockHeight = gvBlockHeight gvList = append(gvList, gv) - if workingBH > gvBlockHeight { + if calcDoneBH > gvBlockHeight { oldGV++ } } diff --git a/core/db_management_test.go b/core/db_management_test.go index 77c9e96..173555b 100644 --- a/core/db_management_test.go +++ b/core/db_management_test.go @@ -1,13 +1,15 @@ package core import ( + "log" + "os" + "path/filepath" + "testing" + "github.com/icon-project/rewardcalculator/common" "github.com/icon-project/rewardcalculator/common/codec" "github.com/icon-project/rewardcalculator/common/db" "github.com/stretchr/testify/assert" - "os" - "path/filepath" - "testing" ) func makeDBInfo() *DBInfo { @@ -16,8 +18,9 @@ func makeDBInfo() *DBInfo { dbInfo.DBRoot = testDBDir dbInfo.DBType = string(db.GoLevelDBBackend) dbInfo.DBCount = 1 - dbInfo.BlockHeight = iaBlockHeight dbInfo.QueryDBIsZero = false + dbInfo.CalcDone = iaBlockHeight + dbInfo.Calculating = iaBlockHeight return dbInfo } @@ -37,8 +40,9 @@ func TestDBMNGDBInfo_BytesAndSetBytes(t *testing.T) { dbInfoNew.SetBytes(bs) assert.Equal(t, dbInfo.DBCount, dbInfoNew.DBCount) - assert.Equal(t, dbInfo.BlockHeight, dbInfoNew.BlockHeight) assert.Equal(t, dbInfo.QueryDBIsZero, dbInfoNew.QueryDBIsZero) + assert.Equal(t, dbInfo.CalcDone, dbInfoNew.CalcDone) + assert.Equal(t, dbInfo.Calculating, dbInfoNew.Calculating) bsNew, _ := dbInfoNew.Bytes() assert.Equal(t, bs, bsNew) } @@ -55,11 +59,11 @@ func TestDBMNGDBInfo_NewDBInfo(t *testing.T) { assert.Equal(t, filepath.Join(testDBDir, testDB), dbInfo.DBRoot) assert.Equal(t, string(db.GoLevelDBBackend), dbInfo.DBType) assert.Equal(t, 1, dbInfo.DBCount) - assert.Equal(t, uint64(0), dbInfo.BlockHeight) - assert.Equal(t, uint64(0), dbInfo.PrevBlockHeight) assert.False(t, dbInfo.QueryDBIsZero) - assert.NotNil(t, dbInfo.BlockHash) - assert.NotNil(t, dbInfo.PrevBlockHash) + assert.True(t, dbInfo.Current.checkValue(uint64(0), zeroBlockHash)) + assert.Equal(t, uint64(0), dbInfo.CalcDone) + assert.Equal(t, uint64(0), dbInfo.PrevCalcDone) + assert.Equal(t, uint64(0), dbInfo.Calculating) bucket, _ := mngDB.GetBucket(db.PrefixManagement) bsNew, err := bucket.Get(dbInfo.ID()) @@ -70,13 +74,11 @@ func TestDBMNGDBInfo_NewDBInfo(t *testing.T) { // update DB Info dbInfo.DBCount = 2 - dbInfo.BlockHeight = 123 dbInfo.QueryDBIsZero = true - blockHash := []byte("blockHash") - copy(dbInfo.BlockHash, blockHash) - dbInfo.PrevBlockHeight = 1 - blockHash = []byte("prevBlockHash") - copy(dbInfo.BlockHash, blockHash) + dbInfo.CalcDone = 200 + dbInfo.PrevCalcDone = 100 + dbInfo.Calculating = dbInfo.CalcDone + bs, _ = dbInfo.Bytes() bucket.Set(dbInfo.ID(), bs) @@ -86,11 +88,11 @@ func TestDBMNGDBInfo_NewDBInfo(t *testing.T) { assert.Equal(t, filepath.Join(testDBDir, testDB), dbInfo1.DBRoot) assert.Equal(t, string(db.GoLevelDBBackend), dbInfo1.DBType) assert.Equal(t, dbInfo.DBCount, dbInfo1.DBCount) - assert.Equal(t, dbInfo.BlockHeight, dbInfo1.BlockHeight) assert.Equal(t, dbInfo.QueryDBIsZero, dbInfo1.QueryDBIsZero) - assert.Equal(t, dbInfo.BlockHash, dbInfo1.BlockHash) - assert.Equal(t, dbInfo.PrevBlockHeight, dbInfo1.PrevBlockHeight) - assert.Equal(t, dbInfo.PrevBlockHash, dbInfo1.PrevBlockHash) + assert.True(t, dbInfo.Current.equal(&dbInfo1.Current)) + assert.Equal(t, dbInfo.CalcDone, dbInfo1.CalcDone) + assert.Equal(t, dbInfo.PrevCalcDone, dbInfo1.PrevCalcDone) + assert.Equal(t, dbInfo.Calculating, dbInfo1.Calculating) } @@ -350,3 +352,28 @@ func TestDBMNGPRepCandidate_LoadPRepCandidate(t *testing.T) { assert.Equal(t, pc.End, pcNew.End) } } + +type dataV1 struct { + DBCount int + BlockHeight uint64 + QueryDBIsZero bool +} + +func TestDBMNGDBInfo_backwardCompatibility(t *testing.T) { + v1 := dataV1{16, 1000, true } + var bytes []byte + if bs, err := codec.MarshalToBytes(&v1); err != nil { + log.Printf("Failed to marshal v1. %v", err) + return + } else { + bytes = bs + } + + var v2 DBInfoDataV2 + + _, err := codec.UnmarshalFromBytes(bytes, &v2) + if err != nil { + log.Printf("Failed to unmarshal v1 with v2. %v", err) + return + } +} diff --git a/core/msg.go b/core/msg.go index e2c4b84..cc11bb6 100644 --- a/core/msg.go +++ b/core/msg.go @@ -1,6 +1,7 @@ package core import ( + "encoding/hex" "encoding/json" "fmt" "github.com/icon-project/rewardcalculator/common" @@ -96,7 +97,8 @@ func newConnection(m *manager, c ipc.Connection) (*msgHandler, error) { } // send READY message to peer - err := sendVersion(c, MsgReady, 0, handler.mgr.ctx.DB.info.BlockHeight) + cBI := handler.mgr.ctx.DB.getCurrentBlockInfo() + err := sendVersion(c, MsgReady, 0, cBI.BlockHeight, cBI.BlockHash) if err != nil { log.Printf("Failed to send READY message") } @@ -137,20 +139,24 @@ func (mh *msgHandler) HandleMessage(c ipc.Connection, msg uint, id uint32, data type ResponseVersion struct { Version uint64 BlockHeight uint64 + BlockHash [BlockHashSize]byte } func (rv *ResponseVersion) String() string { - return fmt.Sprintf("Version: %d, BlockHeight: %d", rv.Version, rv.BlockHeight) + return fmt.Sprintf("Version: %d, BlockHeight: %d, BlockHash: %s", + rv.Version, rv.BlockHeight, hex.EncodeToString(rv.BlockHash[:])) } func (mh *msgHandler) version(c ipc.Connection, id uint32) error { - return sendVersion(c, MsgVersion, id, mh.mgr.ctx.DB.info.BlockHeight) + cBI := mh.mgr.ctx.DB.getCurrentBlockInfo() + return sendVersion(c, MsgVersion, id, cBI.BlockHeight, cBI.BlockHash) } -func sendVersion(c ipc.Connection, msg uint, id uint32, blockHeight uint64) error { +func sendVersion(c ipc.Connection, msg uint, id uint32, blockHeight uint64, blockHash [BlockHashSize]byte) error { resp := ResponseVersion{ Version: IPCVersion, BlockHeight: blockHeight, + BlockHash: blockHash, } log.Printf("Send message. (msg:%s, id:%d, data:%s)", MsgToString(msg), id, resp.String()) diff --git a/core/msg_calculate.go b/core/msg_calculate.go index 60fe2c0..ea0bb8d 100644 --- a/core/msg_calculate.go +++ b/core/msg_calculate.go @@ -1,7 +1,6 @@ package core import ( - "bytes" "encoding/hex" "fmt" "log" @@ -42,14 +41,12 @@ type CalculateRequest struct { } func (cr *CalculateRequest) String() string { - return fmt.Sprintf("Path: %s, BlockHeight: %d, BlockHash: %s", - cr.Path, cr.BlockHeight, hex.EncodeToString(cr.BlockHash)) + return fmt.Sprintf("Path: %s, BlockHeight: %d", cr.Path, cr.BlockHeight) } type CalculateResponse struct { Status uint16 BlockHeight uint64 - BlockHash []byte } const ( @@ -441,7 +438,7 @@ func DoCalculate(quit <-chan struct{}, ctx *Context, req *CalculateRequest, return err, blockHeight, nil, nil } - ctx.DB.setCalculateBlockHeight(req.BlockHeight) + ctx.DB.setCalculatingBH(req.BlockHeight) startTime := time.Now() // open IISS Data @@ -453,29 +450,30 @@ func DoCalculate(quit <-chan struct{}, ctx *Context, req *CalculateRequest, if header == nil { sendCalculateACK(c, id, CalcRespStatusInvalidData, blockHeight) err := fmt.Errorf("Failed to load IISS data (path: %s)\n", req.Path) - ctx.DB.resetCalculateBlockHeight() + ctx.DB.resetCalculatingBH() return err, blockHeight, nil, nil } // set block height blockHeight = header.BlockHeight if blockHeight == 0 { - blockHeight = iScoreDB.info.BlockHeight + 1 + blockHeight = iScoreDB.getCalcDoneBH() + 1 } // check blockHeight and blockHash - if blockHeight == iScoreDB.info.BlockHeight && bytes.Compare(req.BlockHash, iScoreDB.info.BlockHash) == 0 { + calcDoneBH := iScoreDB.getCalcDoneBH() + if calcDoneBH == blockHeight { sendCalculateACK(c, id, CalcRespStatusDuplicateBH, blockHeight) err := fmt.Errorf("duplicated block(height: %d, hash: %s)\n", blockHeight, hex.EncodeToString(req.BlockHash)) - ctx.DB.resetCalculateBlockHeight() + ctx.DB.resetCalculatingBH() return err, blockHeight, nil, nil } - if blockHeight < iScoreDB.info.BlockHeight { + if blockHeight < calcDoneBH { sendCalculateACK(c, id, CalcRespStatusInvalidBH, blockHeight) err := fmt.Errorf("too low blockHeight(request: %d, RC blockHeight: %d)\n", - blockHeight, iScoreDB.info.BlockHeight) - ctx.DB.resetCalculateBlockHeight() + blockHeight, calcDoneBH) + ctx.DB.resetCalculatingBH() return err, blockHeight, nil, nil } @@ -579,12 +577,12 @@ func DoCalculate(quit <-chan struct{}, ctx *Context, req *CalculateRequest, elapsedTime := time.Since(startTime) log.Printf("Finish calculation: Duration: %s, block height: %d -> %d, DB: %d, batch: %d, %d entries", - elapsedTime, ctx.DB.info.BlockHeight, blockHeight, iScoreDB.info.DBCount, writeBatchCount, totalCount) + elapsedTime, ctx.DB.getCalcDoneBH(), blockHeight, iScoreDB.info.DBCount, writeBatchCount, totalCount) log.Printf("%s", stats.String()) log.Printf("stateHash : %s", hex.EncodeToString(stateHash)) // set blockHeight - ctx.DB.setBlockInfo(blockHeight, req.BlockHash) + ctx.DB.setCalcDoneBH(blockHeight) // write calculation result WriteCalculationResult(ctx.DB.getCalculateResultDB(), blockHeight, stats, stateHash) @@ -810,7 +808,7 @@ func calculateIISSBlockProduce(ctx *Context, iissDB db.Database, blockHeight uin func calculatePRepReward(ctx *Context, to uint64) (uint64, *common.HexInt, []byte) { h := sha3.NewShake256() stateHash := make([]byte, 64) - start := ctx.DB.info.BlockHeight + start := ctx.DB.getCalcDoneBH() end := to totalReward := new(common.HexInt) @@ -984,10 +982,10 @@ func (mh *msgHandler) queryCalculateStatus(c ipc.Connection, id uint32, data []b func DoQueryCalculateStatus(ctx *Context, resp *QueryCalculateStatusResponse) { if ctx.DB.isCalculating() { resp.Status = CalculationDoing - resp.BlockHeight = ctx.DB.info.CalcBlockHeight + resp.BlockHeight = ctx.DB.getCalculatingBH() } else { resp.Status = CalculationDone - resp.BlockHeight = ctx.DB.info.BlockHeight + resp.BlockHeight = ctx.DB.getCalcDoneBH() } } @@ -1052,7 +1050,7 @@ func DoQueryCalculateResult(ctx *Context, blockHeight uint64, resp *QueryCalcula // check doing calculation if ctx.DB.isCalculating() { - if blockHeight == ctx.DB.info.CalcBlockHeight { + if blockHeight == ctx.DB.getCalculatingBH() { resp.Status = calcDoing return } diff --git a/core/msg_calculate_test.go b/core/msg_calculate_test.go index 20789b2..35f8119 100644 --- a/core/msg_calculate_test.go +++ b/core/msg_calculate_test.go @@ -635,17 +635,15 @@ func TestMsgCalc_DoCalculate_Error(t *testing.T) { defer finalizeTest(ctx) iissDBDir := testDBDir + "/iiss" - blockHash := make([]byte, BlockHashSize) - copy(blockHash, "test") - req := CalculateRequest{Path: iissDBDir, BlockHeight:100, BlockHash:blockHash} + req := CalculateRequest{Path: iissDBDir, BlockHeight:100, BlockHash:testBlockHash} // get CALCULATE message while processing CALCULATE message - ctx.DB.setCalculateBlockHeight(50) + ctx.DB.setCalculatingBH(uint64(50)) err, blockHeight, _, _ := DoCalculate(ctx.Rollback.GetChannel(), ctx, &req, nil, 0) assert.Error(t, err) assert.True(t, strings.HasPrefix(err.Error(), "calculating now. drop calculate message")) assert.Equal(t, req.BlockHeight, blockHeight) - ctx.DB.resetCalculateBlockHeight() + ctx.DB.resetCalculatingBH() // get CALCULATE message with no IISS data err, blockHeight, _, _ = DoCalculate(ctx.Rollback.GetChannel(), ctx, &req, nil, 0) @@ -659,23 +657,23 @@ func TestMsgCalc_DoCalculate_Error(t *testing.T) { defer os.RemoveAll(iissDBDir) // get CALCULATE message with invalid block height - ctx.DB.setBlockInfo(uint64(200), nil) + ctx.DB.setCalcDoneBH(uint64(200)) err, blockHeight, _, _ = DoCalculate(ctx.Rollback.GetChannel(), ctx, &req, nil, 0) assert.Error(t, err) assert.True(t, strings.HasPrefix(err.Error(), "too low blockHeight")) assert.Equal(t, req.BlockHeight, blockHeight) // get CALCULATE message with duplicated block height - ctx.DB.setBlockInfo(uint64(100), blockHash) - ctx.DB.setCalculateBlockHeight(100) + ctx.DB.setCalcDoneBH(uint64(100)) + ctx.DB.setCalculatingBH(uint64(100)) err, blockHeight, _, _ = DoCalculate(ctx.Rollback.GetChannel(), ctx, &req, nil, 0) assert.Error(t, err) assert.True(t, strings.HasPrefix(err.Error(), "duplicated block")) assert.Equal(t, req.BlockHeight, blockHeight) // Cancel with ROLLBACK - ctx.DB.setBlockInfo(uint64(50), blockHash) - ctx.DB.setCalculateBlockHeight(50) + ctx.DB.setCalcDoneBH(uint64(50)) + ctx.DB.setCalculatingBH(uint64(50)) quitChannel := ctx.Rollback.GetChannel() ctx.Rollback.notifyRollback() @@ -704,14 +702,14 @@ func TestMsgQueryCalc_DoQueryCalculateStatus(t *testing.T) { // start calculation calcBH := uint64(1000) - ctx.DB.setCalculateBlockHeight(calcBH) + ctx.DB.setCalculatingBH(calcBH) DoQueryCalculateStatus(ctx, &resp) assert.Equal(t, CalculationDoing, resp.Status) assert.Equal(t, calcBH, resp.BlockHeight) // end calculation - ctx.DB.setBlockInfo(calcBH, nil) + ctx.DB.setCalcDoneBH(calcBH) DoQueryCalculateStatus(ctx, &resp) assert.Equal(t, CalculationDone, resp.Status) @@ -730,14 +728,14 @@ func TestMsgQueryCalc_DoQueryCalculateResult(t *testing.T) { assert.Equal(t, blockHeight, resp.BlockHeight) // start calculation - ctx.DB.setCalculateBlockHeight(blockHeight) + ctx.DB.setCalculatingBH(blockHeight) DoQueryCalculateResult(ctx, blockHeight, &resp) assert.Equal(t, calcDoing, resp.Status) assert.Equal(t, blockHeight, resp.BlockHeight) // end calculation - ctx.DB.setBlockInfo(blockHeight, nil) + ctx.DB.setCalcDoneBH(blockHeight) crDB := ctx.DB.getCalculateResultDB() stats := new(Statistics) diff --git a/core/msg_claim.go b/core/msg_claim.go index 606868e..1b913ff 100644 --- a/core/msg_claim.go +++ b/core/msg_claim.go @@ -218,6 +218,9 @@ func (mh *msgHandler) commitBlock(c ipc.Connection, id uint32, data []byte) erro if req.Success == true { err = writePreCommitToClaimDB(iDB.getPreCommitDB(), iDB.getClaimDB(), iDB.getClaimBackupDB(), req.BlockHeight, req.BlockHash) + if err == nil { + mh.mgr.ctx.DB.setCurrentBlockInfo(req.BlockHeight, req.BlockHash) + } } else { err = flushPreCommit(iDB.getPreCommitDB(), req.BlockHeight, req.BlockHash) } diff --git a/core/msg_debug.go b/core/msg_debug.go index 21d2ff2..aa87627 100644 --- a/core/msg_debug.go +++ b/core/msg_debug.go @@ -59,7 +59,7 @@ type ResponseDebugStats struct { func handleStats(c ipc.Connection, id uint32, ctx *Context) error { var resp ResponseDebugStats resp.Cmd = DebugStatistics - resp.BlockHeight = ctx.DB.info.BlockHeight + resp.BlockHeight = ctx.DB.getCalcDoneBH() if ctx.stats != nil { resp.Stats = *ctx.stats } diff --git a/core/msg_rollback.go b/core/msg_rollback.go index ffee514..d8f288f 100644 --- a/core/msg_rollback.go +++ b/core/msg_rollback.go @@ -70,10 +70,11 @@ func DoRollBack(ctx *Context, req *RollBackRequest) error { ctx.Rollback.notifyRollback() // must Rollback claim DB first - err = rollbackClaimDB(idb.getClaimDB(), idb.getClaimBackupDB(), blockHeight) + err = rollbackClaimDB(ctx, blockHeight, req.BlockHash) if err != nil { log.Printf("Failed to Rollback claim DB. %+v", err) return err + } else { } if checkAccountDBRollback(ctx, blockHeight) { @@ -89,15 +90,16 @@ func DoRollBack(ctx *Context, req *RollBackRequest) error { func checkRollback(ctx *Context, rollback uint64) (bool, error) { idb := ctx.DB - if idb.info.PrevBlockHeight >= rollback { - return false, &RollbackLowBlockHeightError{idb.info.PrevBlockHeight, rollback} + if idb.getPrevCalcDoneBH() >= rollback { + return false, &RollbackLowBlockHeightError{idb.getPrevCalcDoneBH(), rollback} } return true, nil } func checkAccountDBRollback(ctx *Context, rollback uint64) bool { - if rollback >= ctx.DB.info.CalcBlockHeight { - log.Printf("No need to Rollback account DB. %d >= %d", rollback, ctx.DB.info.CalcBlockHeight) + idb := ctx.DB + if rollback >= idb.getCalcDoneBH() { + log.Printf("No need to Rollback account DB. %d >= %d", rollback, idb.getCalcDoneBH()) return false } diff --git a/core/msg_rollback_test.go b/core/msg_rollback_test.go index aa7af95..43de965 100644 --- a/core/msg_rollback_test.go +++ b/core/msg_rollback_test.go @@ -14,8 +14,8 @@ func TestMsgRollback_checkRollback(t *testing.T) { const calcBlockHeight1 uint64 = 100 const calcBlockHeight2 uint64 = 200 - ctx.DB.setBlockInfo(calcBlockHeight1, []byte(string(calcBlockHeight1))) - ctx.DB.setBlockInfo(calcBlockHeight2, []byte(string(calcBlockHeight2))) + ctx.DB.setCalcDoneBH(calcBlockHeight1) + ctx.DB.setCalcDoneBH(calcBlockHeight2) tests := []struct { name string @@ -68,7 +68,7 @@ func TestMsgRollback_checkAccountDBRollback(t *testing.T) { defer finalizeTest(ctx) const calcBlockHeight uint64 = 100 - ctx.DB.setCalculateBlockHeight(calcBlockHeight) + ctx.DB.setCalcDoneBH(calcBlockHeight) assert.True(t, checkAccountDBRollback(ctx, calcBlockHeight - 1)) assert.False(t, checkAccountDBRollback(ctx, calcBlockHeight)) assert.False(t, checkAccountDBRollback(ctx, calcBlockHeight + 1))