Merge pull request ethereum#64 from maticnetwork/mew-deposits

new: New fetch State sync logic

consensus/bor/bor_test/bor_test.go

@@ -2,45 +2,51 @@ package bortest
 
 import (
 	"encoding/hex"
+	"encoding/json"
+	"fmt"
 	"math/big"
 	"testing"
+	"time"
 
-	"github.com/maticnetwork/bor/common"
 	"github.com/maticnetwork/bor/consensus/bor"
 	"github.com/maticnetwork/bor/core/rawdb"
+
 	"github.com/maticnetwork/bor/crypto"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
 
 	"github.com/maticnetwork/bor/core/types"
 
 	"github.com/maticnetwork/bor/mocks"
 )
 
-func TestCommitSpan(t *testing.T) {
+var (
+	spanPath         = "bor/span/1"
+	clerkPath        = "clerk/event-record/list"
+	clerkQueryParams = "from-time=%d&to-time=%d&page=%d&limit=50"
+)
+
+func TestInsertingSpanSizeBlocks(t *testing.T) {
 	init := buildEthereumInstance(t, rawdb.NewMemoryDatabase())
 	chain := init.ethereum.BlockChain()
 	engine := init.ethereum.Engine()
 	_bor := engine.(*bor.Bor)
-
-	// Mock HeimdallClient.FetchWithRetry to return span data from span.json
-	res, heimdallSpan := loadSpanFromFile(t)
-	h := &mocks.IHeimdallClient{}
-	h.On("FetchWithRetry", "bor", "span", "1").Return(res, nil)
+	h, heimdallSpan := getMockedHeimdallClient(t)
 	_bor.SetHeimdallClient(h)
 
 	db := init.ethereum.ChainDb()
 	block := init.genesis.ToBlock(db)
+	to := int64(block.Header().Time)
+
 	// Insert sprintSize # of blocks so that span is fetched at the start of a new sprint
-	for i := uint64(1); i <= sprintSize; i++ {
+	for i := uint64(1); i <= spanSize; i++ {
 		block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor)
 		insertNewBlock(t, chain, block)
 	}
 
-	// FetchWithRetry is invoked 2 times
-	// 1. bor.FinalizeAndAssemble to prepare a new block when calling buildNextBlock
-	// 2. bor.Finalize via(bc.insertChain => bc.processor.Process)
-	assert.True(t, h.AssertCalled(t, "FetchWithRetry", "bor", "span", "1"))
-	validators, err := _bor.GetCurrentValidators(sprintSize, 256) // new span starts at 256
+	assert.True(t, h.AssertCalled(t, "FetchWithRetry", spanPath, ""))
+	assert.True(t, h.AssertCalled(t, "FetchWithRetry", clerkPath, fmt.Sprintf(clerkQueryParams, 1, to, 1)))
+	validators, err := _bor.GetCurrentValidators(sprintSize, spanSize) // check validator set at the first block of new span
 	if err != nil {
 		t.Fatalf("%s", err)
 	}
@@ -52,73 +58,69 @@ func TestCommitSpan(t *testing.T) {
 	}
 }
 
-func TestIsValidatorAction(t *testing.T) {
+func TestFetchStateSyncEvents(t *testing.T) {
 	init := buildEthereumInstance(t, rawdb.NewMemoryDatabase())
 	chain := init.ethereum.BlockChain()
 	engine := init.ethereum.Engine()
 	_bor := engine.(*bor.Bor)
 
-	proposeStateData, _ := hex.DecodeString("ede01f170000000000000000000000000000000000000000000000000000000000000000")
-	proposeSpanData, _ := hex.DecodeString("4b0e4d17")
-	var tx *types.Transaction
-	tx = types.NewTransaction(
-		0,
-		common.HexToAddress(chain.Config().Bor.StateReceiverContract),
-		big.NewInt(0), 0, big.NewInt(0),
-		proposeStateData,
-	)
-	assert.True(t, _bor.IsValidatorAction(chain, addr, tx))
-
-	tx = types.NewTransaction(
-		0,
-		common.HexToAddress(chain.Config().Bor.ValidatorContract),
-		big.NewInt(0), 0, big.NewInt(0),
-		proposeSpanData,
-	)
-	assert.True(t, _bor.IsValidatorAction(chain, addr, tx))
-
-	res, heimdallSpan := loadSpanFromFile(t)
-	h := &mocks.IHeimdallClient{}
-	h.On("FetchWithRetry", "bor", "span", "1").Return(res, nil)
-	_bor.SetHeimdallClient(h)
-
+	// A. Insert blocks for 0th sprint
 	db := init.ethereum.ChainDb()
 	block := init.genesis.ToBlock(db)
-
-	for i := uint64(1); i <= spanSize; i++ {
+	// Insert sprintSize # of blocks so that span is fetched at the start of a new sprint
+	for i := uint64(1); i < sprintSize; i++ {
 		block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor)
 		insertNewBlock(t, chain, block)
 	}
 
-	for _, validator := range heimdallSpan.SelectedProducers {
-		_addr := validator.Address
-		tx = types.NewTransaction(
-			0,
-			common.HexToAddress(chain.Config().Bor.StateReceiverContract),
-			big.NewInt(0), 0, big.NewInt(0),
-			proposeStateData,
-		)
-		assert.True(t, _bor.IsValidatorAction(chain, _addr, tx))
-
-		tx = types.NewTransaction(
-			0,
-			common.HexToAddress(chain.Config().Bor.ValidatorContract),
-			big.NewInt(0), 0, big.NewInt(0),
-			proposeSpanData,
-		)
-		assert.True(t, _bor.IsValidatorAction(chain, _addr, tx))
+	// B. Before inserting 1st block of the next sprint, mock heimdall deps
+	// B.1 Mock /bor/span/1
+	res, _ := loadSpanFromFile(t)
+	h := &mocks.IHeimdallClient{}
+	h.On("FetchWithRetry", spanPath, "").Return(res, nil)
+
+	// B.2 Mock State Sync events
+	// read heimdall api response from file
+	res = stateSyncEventsPayload(t)
+	var _eventRecords []*bor.EventRecordWithTime
+	if err := json.Unmarshal(res.Result, &_eventRecords); err != nil {
+		t.Fatalf("%s", err)
 	}
+
+	// use that as a sample to generate bor.stateFetchLimit events
+	eventRecords := generateFakeStateSyncEvents(_eventRecords[0], 50)
+	_res, _ := json.Marshal(eventRecords)
+	response := bor.ResponseWithHeight{Height: "0"}
+	if err := json.Unmarshal(_res, &response.Result); err != nil {
+		t.Fatalf("%s", err)
+	}
+
+	// at # sprintSize, events are fetched for the interval [from, (block-sprint).Time)
+	from := 1
+	to := int64(chain.GetHeaderByNumber(0).Time)
+	page := 1
+	query1Params := fmt.Sprintf(clerkQueryParams, from, to, page)
+	h.On("FetchWithRetry", clerkPath, query1Params).Return(&response, nil)
+
+	page = 2
+	query2Params := fmt.Sprintf(clerkQueryParams, from, to, page)
+	h.On("FetchWithRetry", clerkPath, query2Params).Return(&bor.ResponseWithHeight{}, nil)
+	_bor.SetHeimdallClient(h)
+
+	block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor)
+	insertNewBlock(t, chain, block)
+
+	assert.True(t, h.AssertCalled(t, "FetchWithRetry", spanPath, ""))
+	assert.True(t, h.AssertCalled(t, "FetchWithRetry", clerkPath, query1Params))
+	assert.True(t, h.AssertCalled(t, "FetchWithRetry", clerkPath, query2Params))
 }
 
 func TestOutOfTurnSigning(t *testing.T) {
 	init := buildEthereumInstance(t, rawdb.NewMemoryDatabase())
 	chain := init.ethereum.BlockChain()
 	engine := init.ethereum.Engine()
 	_bor := engine.(*bor.Bor)
-
-	res, _ := loadSpanFromFile(t)
-	h := &mocks.IHeimdallClient{}
-	h.On("FetchWithRetry", "bor", "span", "1").Return(res, nil)
+	h, _ := getMockedHeimdallClient(t)
 	_bor.SetHeimdallClient(h)
 
 	db := init.ethereum.ChainDb()
@@ -166,10 +168,7 @@ func TestSignerNotFound(t *testing.T) {
 	chain := init.ethereum.BlockChain()
 	engine := init.ethereum.Engine()
 	_bor := engine.(*bor.Bor)
-
-	res, _ := loadSpanFromFile(t)
-	h := &mocks.IHeimdallClient{}
-	h.On("FetchWithRetry", "bor", "span", "1").Return(res, nil)
+	h, _ := getMockedHeimdallClient(t)
 	_bor.SetHeimdallClient(h)
 
 	db := init.ethereum.ChainDb()
@@ -187,3 +186,27 @@ func TestSignerNotFound(t *testing.T) {
 		*err.(*bor.UnauthorizedSignerError),
 		bor.UnauthorizedSignerError{Number: 0, Signer: addr.Bytes()})
 }
+
+func getMockedHeimdallClient(t *testing.T) (*mocks.IHeimdallClient, *bor.HeimdallSpan) {
+	res, heimdallSpan := loadSpanFromFile(t)
+	h := &mocks.IHeimdallClient{}
+	h.On("FetchWithRetry", "bor/span/1", "").Return(res, nil)
+	h.On("FetchWithRetry", mock.AnythingOfType("string"), mock.AnythingOfType("string")).Return(stateSyncEventsPayload(t), nil)
+	return h, heimdallSpan
+}
+
+func generateFakeStateSyncEvents(sample *bor.EventRecordWithTime, count int) []*bor.EventRecordWithTime {
+	events := make([]*bor.EventRecordWithTime, count)
+	event := *sample
+	event.ID = 0
+	event.Time = time.Now()
+	events[0] = &bor.EventRecordWithTime{}
+	*events[0] = event
+	for i := 1; i < count; i++ {
+		event.ID = uint64(i)
+		event.Time = event.Time.Add(1 * time.Second)
+		events[i] = &bor.EventRecordWithTime{}
+		*events[i] = event
+	}
+	return events
+}

consensus/bor/bor_test/helper.go

@@ -156,6 +156,18 @@ func sign(t *testing.T, header *types.Header, signer []byte) {
 	copy(header.Extra[len(header.Extra)-extraSeal:], sig)
 }
 
+func stateSyncEventsPayload(t *testing.T) *bor.ResponseWithHeight {
+	stateData, err := ioutil.ReadFile("states.json")
+	if err != nil {
+		t.Fatalf("%s", err)
+	}
+	res := &bor.ResponseWithHeight{}
+	if err := json.Unmarshal(stateData, res); err != nil {
+		t.Fatalf("%s", err)
+	}
+	return res
+}
+
 func loadSpanFromFile(t *testing.T) (*bor.ResponseWithHeight, *bor.HeimdallSpan) {
 	spanData, err := ioutil.ReadFile("span.json")
 	if err != nil {

consensus/bor/bor_test/states.json

@@ -0,0 +1,23 @@
+{
+  "height": "0",
+  "result": [
+    {
+      "id": 1,
+      "contract": "0xb55969a6d60413a63291a1de572269875df541e3",
+      "data": "0x00000000000000000000000048aa8d4af32551892fcf08ad63be7dd206d46f6500000000000000000000000048aa8d4af32551892fcf08ad63be7dd206d46f65000000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000000000000000000000000000000000000000014",
+      "tx_hash": "0x7b113e09d98b6d4be1dedfbc0746e34876de767f2cb8b58ff00160a160811dd6",
+      "log_index": 0,
+      "bor_chain_id": "15001",
+      "record_time": "2020-05-15T13:36:38.580995Z"
+    },
+    {
+      "id": 2,
+      "contract": "0xb55969a6d60413a63291a1de572269875df541e3",
+      "data": "0x00000000000000000000000048aa8d4af32551892fcf08ad63be7dd206d46f6500000000000000000000000048aa8d4af32551892fcf08ad63be7dd206d46f65000000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000000000000000000000000000000000000000015",
+      "tx_hash": "0xb72358aff8e4d61f4de97a37a40ddda986c081e0de8036e0a78c4b61b067cba9",
+      "log_index": 0,
+      "bor_chain_id": "15001",
+      "record_time": "2020-05-15T13:42:37.319058Z"
+    }
+  ]
+}

consensus/bor/clerk.go

@@ -1,6 +1,9 @@
 package bor
 
 import (
+	"fmt"
+	"time"
+
 	"github.com/maticnetwork/bor/common"
 	"github.com/maticnetwork/bor/common/hexutil"
 )
@@ -14,3 +17,33 @@ type EventRecord struct {
 	LogIndex uint64         `json:"log_index" yaml:"log_index"`
 	ChainID  string         `json:"bor_chain_id" yaml:"bor_chain_id"`
 }
+
+type EventRecordWithTime struct {
+	EventRecord
+	Time time.Time `json:"record_time" yaml:"record_time"`
+}
+
+// String returns the string representatin of span
+func (e *EventRecordWithTime) String() string {
+	return fmt.Sprintf(
+		"id %v, contract %v, data: %v, txHash: %v, logIndex: %v, chainId: %v, time %s",
+		e.ID,
+		e.Contract.String(),
+		e.Data.String(),
+		e.TxHash.Hex(),
+		e.LogIndex,
+		e.ChainID,
+		e.Time.Format(time.RFC3339),
+	)
+}
+
+func (e *EventRecordWithTime) BuildEventRecord() *EventRecord {
+	return &EventRecord{
+		ID:       e.ID,
+		Contract: e.Contract,
+		Data:     e.Data,
+		TxHash:   e.TxHash,
+		LogIndex: e.LogIndex,
+		ChainID:  e.ChainID,
+	}
+}

consensus/bor/errors.go

@@ -2,6 +2,7 @@ package bor
 
 import (
 	"fmt"
+	"time"
 )
 
 // TotalVotingPowerExceededError is returned when the maximum allowed total voting power is exceeded
@@ -123,3 +124,20 @@ func (e *WrongDifficultyError) Error() string {
 		e.Signer,
 	)
 }
+
+type InvalidStateReceivedError struct {
+	Number uint64
+	From   *time.Time
+	To     *time.Time
+	Event  *EventRecordWithTime
+}
+
+func (e *InvalidStateReceivedError) Error() string {
+	return fmt.Sprintf(
+		"Received event with invalid timestamp at block %d. Requested events from %s to %s. Received %s\n",
+		e.Number,
+		e.From.Format(time.RFC3339),
+		e.To.Format(time.RFC3339),
+		e.Event,
+	)
+}