diff --git a/eth/backend.go b/eth/backend.go
index de40890cf73a..fbff8441033f 100644
--- a/eth/backend.go
+++ b/eth/backend.go
@@ -282,8 +282,8 @@ func New(ctx *node.ServiceContext, config *ethconfig.Config, XDCXServ *XDCx.XDCX
return block, false, nil
}
- eth.protocolManager.fetcher.SetSignHook(signHook)
- eth.protocolManager.fetcher.SetAppendM2HeaderHook(appendM2HeaderHook)
+ eth.protocolManager.blockFetcher.SetSignHook(signHook)
+ eth.protocolManager.blockFetcher.SetAppendM2HeaderHook(appendM2HeaderHook)
/*
XDPoS1.0 Specific hooks
diff --git a/eth/fetcher/fetcher.go b/eth/fetcher/block_fetcher.go
similarity index 74%
rename from eth/fetcher/fetcher.go
rename to eth/fetcher/block_fetcher.go
index 7d1e15fd4dea..642ce6e48fb5 100644
--- a/eth/fetcher/fetcher.go
+++ b/eth/fetcher/block_fetcher.go
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see .
-// Package fetcher contains the block announcement based synchronisation.
+// Package fetcher contains the announcement based blocks or transaction synchronisation.
package fetcher
import (
@@ -29,16 +29,40 @@ import (
"github.com/XinFinOrg/XDPoSChain/consensus"
"github.com/XinFinOrg/XDPoSChain/core/types"
"github.com/XinFinOrg/XDPoSChain/log"
+ "github.com/XinFinOrg/XDPoSChain/metrics"
)
const (
- arriveTimeout = 500 * time.Millisecond // Time allowance before an announced block is explicitly requested
+ arriveTimeout = 500 * time.Millisecond // Time allowance before an announced block/transaction is explicitly requested
gatherSlack = 100 * time.Millisecond // Interval used to collate almost-expired announces with fetches
- fetchTimeout = 5 * time.Second // Maximum allotted time to return an explicitly requested block
- maxUncleDist = 7 // Maximum allowed backward distance from the chain head
- maxQueueDist = 32 // Maximum allowed distance from the chain head to queue
- hashLimit = 256 // Maximum number of unique blocks a peer may have announced
- blockLimit = 64 // Maximum number of unique blocks a peer may have delivered
+ fetchTimeout = 5 * time.Second // Maximum allotted time to return an explicitly requested block/transaction
+)
+
+const (
+ maxUncleDist = 7 // Maximum allowed backward distance from the chain head
+ maxQueueDist = 32 // Maximum allowed distance from the chain head to queue
+ hashLimit = 256 // Maximum number of unique blocks a peer may have announced
+ blockLimit = 64 // Maximum number of unique blocks a peer may have delivered
+)
+
+var (
+ blockAnnounceInMeter = metrics.NewRegisteredMeter("eth/fetcher/block/announces/in", nil)
+ blockAnnounceOutTimer = metrics.NewRegisteredTimer("eth/fetcher/block/announces/out", nil)
+ blockAnnounceDropMeter = metrics.NewRegisteredMeter("eth/fetcher/block/announces/drop", nil)
+ blockAnnounceDOSMeter = metrics.NewRegisteredMeter("eth/fetcher/block/announces/dos", nil)
+
+ blockBroadcastInMeter = metrics.NewRegisteredMeter("eth/fetcher/block/broadcasts/in", nil)
+ blockBroadcastOutTimer = metrics.NewRegisteredTimer("eth/fetcher/block/broadcasts/out", nil)
+ blockBroadcastDropMeter = metrics.NewRegisteredMeter("eth/fetcher/block/broadcasts/drop", nil)
+ blockBroadcastDOSMeter = metrics.NewRegisteredMeter("eth/fetcher/block/broadcasts/dos", nil)
+
+ headerFetchMeter = metrics.NewRegisteredMeter("eth/fetcher/block/headers", nil)
+ bodyFetchMeter = metrics.NewRegisteredMeter("eth/fetcher/block/bodies", nil)
+
+ headerFilterInMeter = metrics.NewRegisteredMeter("eth/fetcher/block/filter/headers/in", nil)
+ headerFilterOutMeter = metrics.NewRegisteredMeter("eth/fetcher/block/filter/headers/out", nil)
+ bodyFilterInMeter = metrics.NewRegisteredMeter("eth/fetcher/block/filter/bodies/in", nil)
+ bodyFilterOutMeter = metrics.NewRegisteredMeter("eth/fetcher/block/filter/bodies/out", nil)
)
var (
@@ -66,17 +90,17 @@ type blockBroadcasterFn func(block *types.Block, propagate bool)
// chainHeightFn is a callback type to retrieve the current chain height.
type chainHeightFn func() uint64
-// blockInsertFn is a callback type to insert a batch of blocks into the local chain.
-type blockInsertFn func(block *types.Block) error
+// chainInsertFn is a callback type to insert a batch of blocks into the local chain.
+type chainInsertFn func(types.Blocks) (int, error)
type blockPrepareFn func(block *types.Block) error
// peerDropFn is a callback type for dropping a peer detected as malicious.
type peerDropFn func(id string)
-// announce is the hash notification of the availability of a new block in the
+// blockAnnounce is the hash notification of the availability of a new block in the
// network.
-type announce struct {
+type blockAnnounce struct {
hash common.Hash // Hash of the block being announced
number uint64 // Number of the block being announced (0 = unknown | old protocol)
header *types.Header // Header of the block partially reassembled (new protocol)
@@ -104,18 +128,18 @@ type bodyFilterTask struct {
time time.Time // Arrival time of the blocks' contents
}
-// inject represents a schedules import operation.
-type inject struct {
+// blockInject represents a schedules import operation.
+type blockInject struct {
origin string
block *types.Block
}
-// Fetcher is responsible for accumulating block announcements from various peers
+// BlockFetcher is responsible for accumulating block announcements from various peers
// and scheduling them for retrieval.
-type Fetcher struct {
+type BlockFetcher struct {
// Various event channels
- notify chan *announce
- inject chan *inject
+ notify chan *blockAnnounce
+ inject chan *blockInject
blockFilter chan chan []*types.Block
headerFilter chan chan *headerFilterTask
@@ -125,16 +149,16 @@ type Fetcher struct {
quit chan struct{}
// Announce states
- announces map[string]int // Per peer announce counts to prevent memory exhaustion
- announced map[common.Hash][]*announce // Announced blocks, scheduled for fetching
- fetching map[common.Hash]*announce // Announced blocks, currently fetching
- fetched map[common.Hash][]*announce // Blocks with headers fetched, scheduled for body retrieval
- completing map[common.Hash]*announce // Blocks with headers, currently body-completing
+ announces map[string]int // Per peer blockAnnounce counts to prevent memory exhaustion
+ announced map[common.Hash][]*blockAnnounce // Announced blocks, scheduled for fetching
+ fetching map[common.Hash]*blockAnnounce // Announced blocks, currently fetching
+ fetched map[common.Hash][]*blockAnnounce // Blocks with headers fetched, scheduled for body retrieval
+ completing map[common.Hash]*blockAnnounce // Blocks with headers, currently body-completing
// Block cache
- queue *prque.Prque // Queue containing the import operations (block number sorted)
- queues map[string]int // Per peer block counts to prevent memory exhaustion
- queued map[common.Hash]*inject // Set of already queued blocks (to dedup imports)
+ queue *prque.Prque // Queue containing the import operations (block number sorted)
+ queues map[string]int // Per peer block counts to prevent memory exhaustion
+ queued map[common.Hash]*blockInject // Set of already queued blocks (to dedupe imports)
knowns *lru.ARCCache
// Callbacks
getBlock blockRetrievalFn // Retrieves a block from the local chain
@@ -142,45 +166,73 @@ type Fetcher struct {
handleProposedBlock proposeBlockHandlerFn // Consensus v2 specific: Hanle new proposed block
broadcastBlock blockBroadcasterFn // Broadcasts a block to connected peers
chainHeight chainHeightFn // Retrieves the current chain's height
- insertBlock blockInsertFn // Injects a batch of blocks into the chain
+ insertChain chainInsertFn // Injects a batch of blocks into the chain
prepareBlock blockPrepareFn
dropPeer peerDropFn // Drops a peer for misbehaving
// Testing hooks
- announceChangeHook func(common.Hash, bool) // Method to call upon adding or deleting a hash from the announce list
+ announceChangeHook func(common.Hash, bool) // Method to call upon adding or deleting a hash from the blockAnnounce list
queueChangeHook func(common.Hash, bool) // Method to call upon adding or deleting a block from the import queue
fetchingHook func([]common.Hash) // Method to call upon starting a block (eth/61) or header (eth/62) fetch
completingHook func([]common.Hash) // Method to call upon starting a block body fetch (eth/62)
+ importedHook func(*types.Block) // Method to call upon successful block import (both eth/61 and eth/62)
+
signHook func(*types.Block) error
appendM2HeaderHook func(*types.Block) (*types.Block, bool, error)
}
-// New creates a block fetcher to retrieve blocks based on hash announcements.
-func New(getBlock blockRetrievalFn, verifyHeader headerVerifierFn, handleProposedBlock proposeBlockHandlerFn, broadcastBlock blockBroadcasterFn, chainHeight chainHeightFn, insertBlock blockInsertFn, prepareBlock blockPrepareFn, dropPeer peerDropFn) *Fetcher {
- knownBlocks, _ := lru.NewARC(blockLimit)
- return &Fetcher{
- notify: make(chan *announce),
- inject: make(chan *inject),
- blockFilter: make(chan chan []*types.Block),
+// // New creates a block fetcher to retrieve blocks based on hash announcements.
+// func New(getBlock blockRetrievalFn, verifyHeader headerVerifierFn, handleProposedBlock proposeBlockHandlerFn, broadcastBlock blockBroadcasterFn, chainHeight chainHeightFn, insertBlock blockInsertFn, prepareBlock blockPrepareFn, dropPeer peerDropFn) *Fetcher {
+// knownBlocks, _ := lru.NewARC(blockLimit)
+// return &Fetcher{
+// notify: make(chan *announce),
+// inject: make(chan *inject),
+// blockFilter: make(chan chan []*types.Block),
+// headerFilter: make(chan chan *headerFilterTask),
+// bodyFilter: make(chan chan *bodyFilterTask),
+// done: make(chan common.Hash),
+// quit: make(chan struct{}),
+// announces: make(map[string]int),
+// announced: make(map[common.Hash][]*announce),
+// fetching: make(map[common.Hash]*announce),
+// fetched: make(map[common.Hash][]*announce),
+// completing: make(map[common.Hash]*announce),
+// queue: prque.New(nil),
+// queues: make(map[string]int),
+// queued: make(map[common.Hash]*inject),
+// knowns: knownBlocks,
+// getBlock: getBlock,
+// verifyHeader: verifyHeader,
+// handleProposedBlock: handleProposedBlock,
+// broadcastBlock: broadcastBlock,
+// chainHeight: chainHeight,
+// insertBlock: insertBlock,
+// prepareBlock: prepareBlock,
+// dropPeer: dropPeer,
+// NewBlockFetcher creates a block fetcher to retrieve blocks based on hash announcements.
+
+func NewBlockFetcher(getBlock blockRetrievalFn, verifyHeader headerVerifierFn, handleProposedBlock proposeBlockHandlerFn, broadcastBlock blockBroadcasterFn, chainHeight chainHeightFn, insertChain chainInsertFn, prepareBlock blockPrepareFn, dropPeer peerDropFn) *BlockFetcher {
+ return &BlockFetcher{
+ notify: make(chan *blockAnnounce),
+ inject: make(chan *blockInject),
headerFilter: make(chan chan *headerFilterTask),
bodyFilter: make(chan chan *bodyFilterTask),
done: make(chan common.Hash),
quit: make(chan struct{}),
announces: make(map[string]int),
- announced: make(map[common.Hash][]*announce),
- fetching: make(map[common.Hash]*announce),
- fetched: make(map[common.Hash][]*announce),
- completing: make(map[common.Hash]*announce),
+ announced: make(map[common.Hash][]*blockAnnounce),
+ fetching: make(map[common.Hash]*blockAnnounce),
+ fetched: make(map[common.Hash][]*blockAnnounce),
+ completing: make(map[common.Hash]*blockAnnounce),
queue: prque.New(nil),
queues: make(map[string]int),
- queued: make(map[common.Hash]*inject),
- knowns: knownBlocks,
+ queued: make(map[common.Hash]*blockInject),
getBlock: getBlock,
verifyHeader: verifyHeader,
handleProposedBlock: handleProposedBlock,
broadcastBlock: broadcastBlock,
chainHeight: chainHeight,
- insertBlock: insertBlock,
+ insertChain: insertChain,
prepareBlock: prepareBlock,
dropPeer: dropPeer,
}
@@ -188,21 +240,21 @@ func New(getBlock blockRetrievalFn, verifyHeader headerVerifierFn, handlePropose
// Start boots up the announcement based synchroniser, accepting and processing
// hash notifications and block fetches until termination requested.
-func (f *Fetcher) Start() {
+func (f *BlockFetcher) Start() {
go f.loop()
}
// Stop terminates the announcement based synchroniser, canceling all pending
// operations.
-func (f *Fetcher) Stop() {
+func (f *BlockFetcher) Stop() {
close(f.quit)
}
// Notify announces the fetcher of the potential availability of a new block in
// the network.
-func (f *Fetcher) Notify(peer string, hash common.Hash, number uint64, time time.Time,
+func (f *BlockFetcher) Notify(peer string, hash common.Hash, number uint64, time time.Time,
headerFetcher headerRequesterFn, bodyFetcher bodyRequesterFn) error {
- block := &announce{
+ block := &blockAnnounce{
hash: hash,
number: number,
time: time,
@@ -218,9 +270,9 @@ func (f *Fetcher) Notify(peer string, hash common.Hash, number uint64, time time
}
}
-// Enqueue tries to fill gaps the the fetcher's future import queue.
-func (f *Fetcher) Enqueue(peer string, block *types.Block) error {
- op := &inject{
+// Enqueue tries to fill gaps the fetcher's future import queue.
+func (f *BlockFetcher) Enqueue(peer string, block *types.Block) error {
+ op := &blockInject{
origin: peer,
block: block,
}
@@ -234,7 +286,7 @@ func (f *Fetcher) Enqueue(peer string, block *types.Block) error {
// FilterHeaders extracts all the headers that were explicitly requested by the fetcher,
// returning those that should be handled differently.
-func (f *Fetcher) FilterHeaders(peer string, headers []*types.Header, time time.Time) []*types.Header {
+func (f *BlockFetcher) FilterHeaders(peer string, headers []*types.Header, time time.Time) []*types.Header {
log.Trace("Filtering headers", "peer", peer, "headers", len(headers))
// Send the filter channel to the fetcher
@@ -262,7 +314,7 @@ func (f *Fetcher) FilterHeaders(peer string, headers []*types.Header, time time.
// FilterBodies extracts all the block bodies that were explicitly requested by
// the fetcher, returning those that should be handled differently.
-func (f *Fetcher) FilterBodies(peer string, transactions [][]*types.Transaction, uncles [][]*types.Header, time time.Time) ([][]*types.Transaction, [][]*types.Header) {
+func (f *BlockFetcher) FilterBodies(peer string, transactions [][]*types.Transaction, uncles [][]*types.Header, time time.Time) ([][]*types.Transaction, [][]*types.Header) {
log.Trace("Filtering bodies", "peer", peer, "txs", len(transactions), "uncles", len(uncles))
// Send the filter channel to the fetcher
@@ -290,7 +342,7 @@ func (f *Fetcher) FilterBodies(peer string, transactions [][]*types.Transaction,
// Loop is the main fetcher loop, checking and processing various notification
// events.
-func (f *Fetcher) loop() {
+func (f *BlockFetcher) loop() {
// Iterate the block fetching until a quit is requested
fetchTimer := time.NewTimer(0)
completeTimer := time.NewTimer(0)
@@ -305,48 +357,49 @@ func (f *Fetcher) loop() {
// Import any queued blocks that could potentially fit
height := f.chainHeight()
for !f.queue.Empty() {
- op := f.queue.PopItem().(*inject)
+ op := f.queue.PopItem().(*blockInject)
+ hash := op.block.Hash()
if f.queueChangeHook != nil {
- f.queueChangeHook(op.block.Hash(), false)
+ f.queueChangeHook(hash, false)
}
// If too high up the chain or phase, continue later
number := op.block.NumberU64()
if number > height+1 {
- f.queue.Push(op, -int64(op.block.NumberU64()))
+ f.queue.Push(op, -int64(number))
if f.queueChangeHook != nil {
- f.queueChangeHook(op.block.Hash(), true)
+ f.queueChangeHook(hash, true)
}
break
}
// Otherwise if fresh and still unknown, try and import
- hash := op.block.Hash()
if number+maxUncleDist < height || f.getBlock(hash) != nil {
f.forgetBlock(hash)
continue
}
f.insert(op.origin, op.block)
}
+
// Wait for an outside event to occur
select {
case <-f.quit:
- // Fetcher terminating, abort all operations
+ // BlockFetcher terminating, abort all operations
return
case notification := <-f.notify:
// A block was announced, make sure the peer isn't DOSing us
- propAnnounceInMeter.Mark(1)
+ blockAnnounceInMeter.Mark(1)
count := f.announces[notification.origin] + 1
if count > hashLimit {
log.Debug("Peer exceeded outstanding announces", "peer", notification.origin, "limit", hashLimit)
- propAnnounceDOSMeter.Mark(1)
+ blockAnnounceDOSMeter.Mark(1)
break
}
// If we have a valid block number, check that it's potentially useful
if notification.number > 0 {
if dist := int64(notification.number) - int64(f.chainHeight()); dist < -maxUncleDist || dist > maxQueueDist {
log.Debug("Peer discarded announcement", "peer", notification.origin, "number", notification.number, "hash", notification.hash, "distance", dist)
- propAnnounceDropMeter.Mark(1)
+ blockAnnounceDropMeter.Mark(1)
break
}
}
@@ -368,7 +421,7 @@ func (f *Fetcher) loop() {
case op := <-f.inject:
// A direct block insertion was requested, try and fill any pending gaps
- propBroadcastInMeter.Mark(1)
+ blockBroadcastInMeter.Mark(1)
f.enqueue(op.origin, op.block)
case hash := <-f.done:
@@ -454,8 +507,8 @@ func (f *Fetcher) loop() {
headerFilterInMeter.Mark(int64(len(task.headers)))
// Split the batch of headers into unknown ones (to return to the caller),
- // knowns incomplete ones (requiring body retrievals) and completed blocks.
- unknown, incomplete, complete := []*types.Header{}, []*announce{}, []*types.Block{}
+ // known incomplete ones (requiring body retrievals) and completed blocks.
+ unknown, incomplete, complete := []*types.Header{}, []*blockAnnounce{}, []*types.Block{}
for _, header := range task.headers {
hash := header.Hash()
@@ -491,7 +544,7 @@ func (f *Fetcher) loop() {
f.forgetHash(hash)
}
} else {
- // Fetcher doesn't know about it, add to the return list
+ // BlockFetcher doesn't know about it, add to the return list
unknown = append(unknown, header)
}
}
@@ -578,8 +631,8 @@ func (f *Fetcher) loop() {
}
}
-// rescheduleFetch resets the specified fetch timer to the next announce timeout.
-func (f *Fetcher) rescheduleFetch(fetch *time.Timer) {
+// rescheduleFetch resets the specified fetch timer to the next blockAnnounce timeout.
+func (f *BlockFetcher) rescheduleFetch(fetch *time.Timer) {
// Short circuit if no blocks are announced
if len(f.announced) == 0 {
return
@@ -595,7 +648,7 @@ func (f *Fetcher) rescheduleFetch(fetch *time.Timer) {
}
// rescheduleComplete resets the specified completion timer to the next fetch timeout.
-func (f *Fetcher) rescheduleComplete(complete *time.Timer) {
+func (f *BlockFetcher) rescheduleComplete(complete *time.Timer) {
// Short circuit if no headers are fetched
if len(f.fetched) == 0 {
return
@@ -612,7 +665,7 @@ func (f *Fetcher) rescheduleComplete(complete *time.Timer) {
// enqueue schedules a new future import operation, if the block to be imported
// has not yet been seen.
-func (f *Fetcher) enqueue(peer string, block *types.Block) {
+func (f *BlockFetcher) enqueue(peer string, block *types.Block) {
hash := block.Hash()
if f.knowns.Contains(hash) {
log.Trace("Discarded propagated block, known block", "peer", peer, "number", block.Number(), "hash", hash, "limit", blockLimit)
@@ -622,20 +675,20 @@ func (f *Fetcher) enqueue(peer string, block *types.Block) {
count := f.queues[peer] + 1
if count > blockLimit {
log.Debug("Discarded propagated block, exceeded allowance", "peer", peer, "number", block.Number(), "hash", hash, "limit", blockLimit)
- propBroadcastDOSMeter.Mark(1)
+ blockBroadcastDOSMeter.Mark(1)
f.forgetHash(hash)
return
}
// Discard any past or too distant blocks
if dist := int64(block.NumberU64()) - int64(f.chainHeight()); dist < -maxUncleDist || dist > maxQueueDist {
log.Debug("Discarded propagated block, too far away", "peer", peer, "number", block.Number(), "hash", hash, "distance", dist)
- propBroadcastDropMeter.Mark(1)
+ blockBroadcastDropMeter.Mark(1)
f.forgetHash(hash)
return
}
// Schedule the block for future importing
if _, ok := f.queued[hash]; !ok {
- op := &inject{
+ op := &blockInject{
origin: peer,
block: block,
}
@@ -653,7 +706,7 @@ func (f *Fetcher) enqueue(peer string, block *types.Block) {
// insert spawns a new goroutine to run a block insertion into the chain. If the
// block's number is at the same height as the current import phase, it updates
// the phase states accordingly.
-func (f *Fetcher) insert(peer string, block *types.Block) {
+func (f *BlockFetcher) insert(peer string, block *types.Block) {
hash := block.Hash()
// Run the import on a new thread
@@ -667,17 +720,18 @@ func (f *Fetcher) insert(peer string, block *types.Block) {
log.Debug("Unknown parent of propagated block", "peer", peer, "number", block.Number(), "hash", hash, "parent", block.ParentHash())
return
}
- fastBroadCast := true
+ fastBroadCast := true //TODO: double check if we need fastBroadCast logic
again:
err := f.verifyHeader(block.Header())
// Quickly validate the header and propagate the block if it passes
switch err {
case nil:
// All ok, quickly propagate to our peers
- propBroadcastOutTimer.UpdateSince(block.ReceivedAt)
+ blockBroadcastOutTimer.UpdateSince(block.ReceivedAt)
if fastBroadCast {
go f.broadcastBlock(block, true)
}
+
case consensus.ErrFutureBlock:
delay := time.Unix(block.Time().Int64(), 0).Sub(time.Now()) // nolint: gosimple
log.Info("Receive future block", "number", block.NumberU64(), "hash", block.Hash().Hex(), "delay", delay)
@@ -708,7 +762,7 @@ func (f *Fetcher) insert(peer string, block *types.Block) {
}
block = newBlock
fastBroadCast = false
- goto again
+ goto again //TODO: doublecheck if goto again logic is required
default:
// Something went very wrong, drop the peer
log.Warn("Propagated block verification failed", "peer", peer, "number", block.Number(), "hash", hash, "err", err)
@@ -716,7 +770,7 @@ func (f *Fetcher) insert(peer string, block *types.Block) {
return
}
// Run the actual import and log any issues
- if err := f.insertBlock(block); err != nil {
+ if _, err := f.insertChain(types.Blocks{block}); err != nil {
log.Warn("Propagated block import failed", "peer", peer, "number", block.Number(), "hash", hash, "err", err)
return
}
@@ -732,16 +786,21 @@ func (f *Fetcher) insert(peer string, block *types.Block) {
log.Warn("[insert] Unable to handle new proposed block", "err", err, "number", block.Number(), "hash", block.Hash())
}
// If import succeeded, broadcast the block
- propAnnounceOutTimer.UpdateSince(block.ReceivedAt)
+ blockAnnounceOutTimer.UpdateSince(block.ReceivedAt)
if !fastBroadCast {
- go f.broadcastBlock(block, true)
+ go f.broadcastBlock(block, false)
+ }
+
+ // Invoke the testing hook if needed
+ if f.importedHook != nil {
+ f.importedHook(block)
}
}()
}
// forgetHash removes all traces of a block announcement from the fetcher's
// internal state.
-func (f *Fetcher) forgetHash(hash common.Hash) {
+func (f *BlockFetcher) forgetHash(hash common.Hash) {
// Remove all pending announces and decrement DOS counters
for _, announce := range f.announced[hash] {
f.announces[announce.origin]--
@@ -783,7 +842,7 @@ func (f *Fetcher) forgetHash(hash common.Hash) {
// forgetBlock removes all traces of a queued block from the fetcher's internal
// state.
-func (f *Fetcher) forgetBlock(hash common.Hash) {
+func (f *BlockFetcher) forgetBlock(hash common.Hash) {
if insert := f.queued[hash]; insert != nil {
f.queues[insert.origin]--
if f.queues[insert.origin] == 0 {
@@ -794,11 +853,11 @@ func (f *Fetcher) forgetBlock(hash common.Hash) {
}
// Bind double validate hook before block imported into chain.
-func (f *Fetcher) SetSignHook(signHook func(*types.Block) error) {
+func (f *BlockFetcher) SetSignHook(signHook func(*types.Block) error) {
f.signHook = signHook
}
// Bind append m2 to block header hook when imported into chain.
-func (f *Fetcher) SetAppendM2HeaderHook(appendM2HeaderHook func(*types.Block) (*types.Block, bool, error)) {
+func (f *BlockFetcher) SetAppendM2HeaderHook(appendM2HeaderHook func(*types.Block) (*types.Block, bool, error)) {
f.appendM2HeaderHook = appendM2HeaderHook
}
diff --git a/eth/fetcher/fetcher_test.go b/eth/fetcher/block_fetcher_test.go
similarity index 98%
rename from eth/fetcher/fetcher_test.go
rename to eth/fetcher/block_fetcher_test.go
index 484d62d87243..21e0d45766bc 100644
--- a/eth/fetcher/fetcher_test.go
+++ b/eth/fetcher/block_fetcher_test.go
@@ -77,7 +77,7 @@ func makeChain(n int, seed byte, parent *types.Block) ([]common.Hash, map[common
// fetcherTester is a test simulator for mocking out local block chain.
type fetcherTester struct {
- fetcher *Fetcher
+ fetcher *BlockFetcher
hashes []common.Hash // Hash chain belonging to the tester
blocks map[common.Hash]*types.Block // Blocks belonging to the tester
@@ -93,7 +93,7 @@ func newTester() *fetcherTester {
blocks: map[common.Hash]*types.Block{genesis.Hash(): genesis},
drops: make(map[string]bool),
}
- tester.fetcher = New(tester.getBlock, tester.verifyHeader, tester.handleProposedBlock, tester.broadcastBlock, tester.chainHeight, tester.insertBlock, tester.prepareBlock, tester.dropPeer)
+ tester.fetcher = NewBlockFetcher(tester.getBlock, tester.verifyHeader, tester.handleProposedBlock, tester.broadcastBlock, tester.chainHeight, tester.insertChain, tester.prepareBlock, tester.dropPeer)
tester.fetcher.Start()
return tester
@@ -556,9 +556,9 @@ func testImportDeduplication(t *testing.T, protocol int) {
bodyFetcher := tester.makeBodyFetcher("valid", blocks, 0)
counter := uint32(0)
- tester.fetcher.insertBlock = func(block *types.Block) error {
+ tester.fetcher.insertChain = func(blocks types.Blocks) (int, error) {
atomic.AddUint32(&counter, uint32(1))
- return tester.insertBlock(block)
+ return tester.insertChain(blocks)
}
// Instrument the fetching and imported events
fetching := make(chan []common.Hash)
diff --git a/eth/fetcher/metrics.go b/eth/fetcher/metrics.go
deleted file mode 100644
index eb4745904ae8..000000000000
--- a/eth/fetcher/metrics.go
+++ /dev/null
@@ -1,43 +0,0 @@
-// Copyright 2015 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see .
-
-// Contains the metrics collected by the fetcher.
-
-package fetcher
-
-import (
- "github.com/XinFinOrg/XDPoSChain/metrics"
-)
-
-var (
- propAnnounceInMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/announces/in", nil)
- propAnnounceOutTimer = metrics.NewRegisteredTimer("eth/fetcher/prop/announces/out", nil)
- propAnnounceDropMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/announces/drop", nil)
- propAnnounceDOSMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/announces/dos", nil)
-
- propBroadcastInMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/broadcasts/in", nil)
- propBroadcastOutTimer = metrics.NewRegisteredTimer("eth/fetcher/prop/broadcasts/out", nil)
- propBroadcastDropMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/broadcasts/drop", nil)
- propBroadcastDOSMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/broadcasts/dos", nil)
-
- headerFetchMeter = metrics.NewRegisteredMeter("eth/fetcher/fetch/headers", nil)
- bodyFetchMeter = metrics.NewRegisteredMeter("eth/fetcher/fetch/bodies", nil)
-
- headerFilterInMeter = metrics.NewRegisteredMeter("eth/fetcher/filter/headers/in", nil)
- headerFilterOutMeter = metrics.NewRegisteredMeter("eth/fetcher/filter/headers/out", nil)
- bodyFilterInMeter = metrics.NewRegisteredMeter("eth/fetcher/filter/bodies/in", nil)
- bodyFilterOutMeter = metrics.NewRegisteredMeter("eth/fetcher/filter/bodies/out", nil)
-)
diff --git a/eth/fetcher/tx_fetcher.go b/eth/fetcher/tx_fetcher.go
new file mode 100644
index 000000000000..208a88cf097d
--- /dev/null
+++ b/eth/fetcher/tx_fetcher.go
@@ -0,0 +1,894 @@
+// Copyright 2020 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package fetcher
+
+import (
+ "bytes"
+ "fmt"
+ mrand "math/rand"
+ "sort"
+ "time"
+
+ mapset "github.com/deckarep/golang-set"
+ "github.com/XinFinOrg/XDPoSChain/common"
+ "github.com/XinFinOrg/XDPoSChain/common/mclock"
+ "github.com/XinFinOrg/XDPoSChain/core"
+ "github.com/XinFinOrg/XDPoSChain/core/types"
+ "github.com/XinFinOrg/XDPoSChain/log"
+ "github.com/XinFinOrg/XDPoSChain/metrics"
+)
+
+const (
+ // maxTxAnnounces is the maximum number of unique transaction a peer
+ // can announce in a short time.
+ maxTxAnnounces = 4096
+
+ // maxTxRetrievals is the maximum transaction number can be fetched in one
+ // request. The rationale to pick 256 is:
+ // - In eth protocol, the softResponseLimit is 2MB. Nowadays according to
+ // Etherscan the average transaction size is around 200B, so in theory
+ // we can include lots of transaction in a single protocol packet.
+ // - However the maximum size of a single transaction is raised to 128KB,
+ // so pick a middle value here to ensure we can maximize the efficiency
+ // of the retrieval and response size overflow won't happen in most cases.
+ maxTxRetrievals = 256
+
+ // maxTxUnderpricedSetSize is the size of the underpriced transaction set that
+ // is used to track recent transactions that have been dropped so we don't
+ // re-request them.
+ maxTxUnderpricedSetSize = 32768
+
+ // txArriveTimeout is the time allowance before an announced transaction is
+ // explicitly requested.
+ txArriveTimeout = 500 * time.Millisecond
+
+ // txGatherSlack is the interval used to collate almost-expired announces
+ // with network fetches.
+ txGatherSlack = 100 * time.Millisecond
+)
+
+var (
+ // txFetchTimeout is the maximum allotted time to return an explicitly
+ // requested transaction.
+ txFetchTimeout = 5 * time.Second
+)
+
+var (
+ txAnnounceInMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/announces/in", nil)
+ txAnnounceKnownMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/announces/known", nil)
+ txAnnounceUnderpricedMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/announces/underpriced", nil)
+ txAnnounceDOSMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/announces/dos", nil)
+
+ txBroadcastInMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/broadcasts/in", nil)
+ txBroadcastKnownMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/broadcasts/known", nil)
+ txBroadcastUnderpricedMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/broadcasts/underpriced", nil)
+ txBroadcastOtherRejectMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/broadcasts/otherreject", nil)
+
+ txRequestOutMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/request/out", nil)
+ txRequestFailMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/request/fail", nil)
+ txRequestDoneMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/request/done", nil)
+ txRequestTimeoutMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/request/timeout", nil)
+
+ txReplyInMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/replies/in", nil)
+ txReplyKnownMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/replies/known", nil)
+ txReplyUnderpricedMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/replies/underpriced", nil)
+ txReplyOtherRejectMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/replies/otherreject", nil)
+
+ txFetcherWaitingPeers = metrics.NewRegisteredGauge("eth/fetcher/transaction/waiting/peers", nil)
+ txFetcherWaitingHashes = metrics.NewRegisteredGauge("eth/fetcher/transaction/waiting/hashes", nil)
+ txFetcherQueueingPeers = metrics.NewRegisteredGauge("eth/fetcher/transaction/queueing/peers", nil)
+ txFetcherQueueingHashes = metrics.NewRegisteredGauge("eth/fetcher/transaction/queueing/hashes", nil)
+ txFetcherFetchingPeers = metrics.NewRegisteredGauge("eth/fetcher/transaction/fetching/peers", nil)
+ txFetcherFetchingHashes = metrics.NewRegisteredGauge("eth/fetcher/transaction/fetching/hashes", nil)
+)
+
+// txAnnounce is the notification of the availability of a batch
+// of new transactions in the network.
+type txAnnounce struct {
+ origin string // Identifier of the peer originating the notification
+ hashes []common.Hash // Batch of transaction hashes being announced
+}
+
+// txRequest represents an in-flight transaction retrieval request destined to
+// a specific peers.
+type txRequest struct {
+ hashes []common.Hash // Transactions having been requested
+ stolen map[common.Hash]struct{} // Deliveries by someone else (don't re-request)
+ time mclock.AbsTime // Timestamp of the request
+}
+
+// txDelivery is the notification that a batch of transactions have been added
+// to the pool and should be untracked.
+type txDelivery struct {
+ origin string // Identifier of the peer originating the notification
+ hashes []common.Hash // Batch of transaction hashes having been delivered
+ direct bool // Whether this is a direct reply or a broadcast
+}
+
+// txDrop is the notiication that a peer has disconnected.
+type txDrop struct {
+ peer string
+}
+
+// TxFetcher is responsible for retrieving new transaction based on announcements.
+//
+// The fetcher operates in 3 stages:
+// - Transactions that are newly discovered are moved into a wait list.
+// - After ~500ms passes, transactions from the wait list that have not been
+// broadcast to us in whole are moved into a queueing area.
+// - When a connected peer doesn't have in-flight retrieval requests, any
+// transaction queued up (and announced by the peer) are allocated to the
+// peer and moved into a fetching status until it's fulfilled or fails.
+//
+// The invariants of the fetcher are:
+// - Each tracked transaction (hash) must only be present in one of the
+// three stages. This ensures that the fetcher operates akin to a finite
+// state automata and there's do data leak.
+// - Each peer that announced transactions may be scheduled retrievals, but
+// only ever one concurrently. This ensures we can immediately know what is
+// missing from a reply and reschedule it.
+type TxFetcher struct {
+ notify chan *txAnnounce
+ cleanup chan *txDelivery
+ drop chan *txDrop
+ quit chan struct{}
+
+ underpriced mapset.Set // Transactions discarded as too cheap (don't re-fetch)
+
+ // Stage 1: Waiting lists for newly discovered transactions that might be
+ // broadcast without needing explicit request/reply round trips.
+ waitlist map[common.Hash]map[string]struct{} // Transactions waiting for an potential broadcast
+ waittime map[common.Hash]mclock.AbsTime // Timestamps when transactions were added to the waitlist
+ waitslots map[string]map[common.Hash]struct{} // Waiting announcement sgroupped by peer (DoS protection)
+
+ // Stage 2: Queue of transactions that waiting to be allocated to some peer
+ // to be retrieved directly.
+ announces map[string]map[common.Hash]struct{} // Set of announced transactions, grouped by origin peer
+ announced map[common.Hash]map[string]struct{} // Set of download locations, grouped by transaction hash
+
+ // Stage 3: Set of transactions currently being retrieved, some which may be
+ // fulfilled and some rescheduled. Note, this step shares 'announces' from the
+ // previous stage to avoid having to duplicate (need it for DoS checks).
+ fetching map[common.Hash]string // Transaction set currently being retrieved
+ requests map[string]*txRequest // In-flight transaction retrievals
+ alternates map[common.Hash]map[string]struct{} // In-flight transaction alternate origins if retrieval fails
+
+ // Callbacks
+ hasTx func(common.Hash) bool // Retrieves a tx from the local txpool
+ addTxs func([]*types.Transaction) []error // Insert a batch of transactions into local txpool
+ fetchTxs func(string, []common.Hash) error // Retrieves a set of txs from a remote peer
+
+ step chan struct{} // Notification channel when the fetcher loop iterates
+ clock mclock.Clock // Time wrapper to simulate in tests
+ rand *mrand.Rand // Randomizer to use in tests instead of map range loops (soft-random)
+}
+
+// NewTxFetcher creates a transaction fetcher to retrieve transaction
+// based on hash announcements.
+func NewTxFetcher(hasTx func(common.Hash) bool, addTxs func([]*types.Transaction) []error, fetchTxs func(string, []common.Hash) error) *TxFetcher {
+ return NewTxFetcherForTests(hasTx, addTxs, fetchTxs, mclock.System{}, nil)
+}
+
+// NewTxFetcherForTests is a testing method to mock out the realtime clock with
+// a simulated version and the internal randomness with a deterministic one.
+func NewTxFetcherForTests(
+ hasTx func(common.Hash) bool, addTxs func([]*types.Transaction) []error, fetchTxs func(string, []common.Hash) error,
+ clock mclock.Clock, rand *mrand.Rand) *TxFetcher {
+ return &TxFetcher{
+ notify: make(chan *txAnnounce),
+ cleanup: make(chan *txDelivery),
+ drop: make(chan *txDrop),
+ quit: make(chan struct{}),
+ waitlist: make(map[common.Hash]map[string]struct{}),
+ waittime: make(map[common.Hash]mclock.AbsTime),
+ waitslots: make(map[string]map[common.Hash]struct{}),
+ announces: make(map[string]map[common.Hash]struct{}),
+ announced: make(map[common.Hash]map[string]struct{}),
+ fetching: make(map[common.Hash]string),
+ requests: make(map[string]*txRequest),
+ alternates: make(map[common.Hash]map[string]struct{}),
+ underpriced: mapset.NewSet(),
+ hasTx: hasTx,
+ addTxs: addTxs,
+ fetchTxs: fetchTxs,
+ clock: clock,
+ rand: rand,
+ }
+}
+
+// Notify announces the fetcher of the potential availability of a new batch of
+// transactions in the network.
+func (f *TxFetcher) Notify(peer string, hashes []common.Hash) error {
+ // Keep track of all the announced transactions
+ txAnnounceInMeter.Mark(int64(len(hashes)))
+
+ // Skip any transaction announcements that we already know of, or that we've
+ // previously marked as cheap and discarded. This check is of course racey,
+ // because multiple concurrent notifies will still manage to pass it, but it's
+ // still valuable to check here because it runs concurrent to the internal
+ // loop, so anything caught here is time saved internally.
+ var (
+ unknowns = make([]common.Hash, 0, len(hashes))
+ duplicate, underpriced int64
+ )
+ for _, hash := range hashes {
+ switch {
+ case f.hasTx(hash):
+ duplicate++
+
+ case f.underpriced.Contains(hash):
+ underpriced++
+
+ default:
+ unknowns = append(unknowns, hash)
+ }
+ }
+ txAnnounceKnownMeter.Mark(duplicate)
+ txAnnounceUnderpricedMeter.Mark(underpriced)
+
+ // If anything's left to announce, push it into the internal loop
+ if len(unknowns) == 0 {
+ return nil
+ }
+ announce := &txAnnounce{
+ origin: peer,
+ hashes: unknowns,
+ }
+ select {
+ case f.notify <- announce:
+ return nil
+ case <-f.quit:
+ return errTerminated
+ }
+}
+
+// Enqueue imports a batch of received transaction into the transaction pool
+// and the fetcher. This method may be called by both transaction broadcasts and
+// direct request replies. The differentiation is important so the fetcher can
+// re-shedule missing transactions as soon as possible.
+func (f *TxFetcher) Enqueue(peer string, txs []*types.Transaction, direct bool) error {
+ // Keep track of all the propagated transactions
+ if direct {
+ txReplyInMeter.Mark(int64(len(txs)))
+ } else {
+ txBroadcastInMeter.Mark(int64(len(txs)))
+ }
+ // Push all the transactions into the pool, tracking underpriced ones to avoid
+ // re-requesting them and dropping the peer in case of malicious transfers.
+ var (
+ added = make([]common.Hash, 0, len(txs))
+ duplicate int64
+ underpriced int64
+ otherreject int64
+ )
+ errs := f.addTxs(txs)
+ for i, err := range errs {
+ if err != nil {
+ // Track the transaction hash if the price is too low for us.
+ // Avoid re-request this transaction when we receive another
+ // announcement.
+ if err == core.ErrUnderpriced || err == core.ErrReplaceUnderpriced {
+ for f.underpriced.Cardinality() >= maxTxUnderpricedSetSize {
+ f.underpriced.Pop()
+ }
+ f.underpriced.Add(txs[i].Hash())
+ }
+ // Track a few interesting failure types
+ switch err {
+ case nil: // Noop, but need to handle to not count these
+
+ case core.ErrAlreadyKnown:
+ duplicate++
+
+ case core.ErrUnderpriced, core.ErrReplaceUnderpriced:
+ underpriced++
+
+ default:
+ otherreject++
+ }
+ }
+ added = append(added, txs[i].Hash())
+ }
+ if direct {
+ txReplyKnownMeter.Mark(duplicate)
+ txReplyUnderpricedMeter.Mark(underpriced)
+ txReplyOtherRejectMeter.Mark(otherreject)
+ } else {
+ txBroadcastKnownMeter.Mark(duplicate)
+ txBroadcastUnderpricedMeter.Mark(underpriced)
+ txBroadcastOtherRejectMeter.Mark(otherreject)
+ }
+ select {
+ case f.cleanup <- &txDelivery{origin: peer, hashes: added, direct: direct}:
+ return nil
+ case <-f.quit:
+ return errTerminated
+ }
+}
+
+// Drop should be called when a peer disconnects. It cleans up all the internal
+// data structures of the given node.
+func (f *TxFetcher) Drop(peer string) error {
+ select {
+ case f.drop <- &txDrop{peer: peer}:
+ return nil
+ case <-f.quit:
+ return errTerminated
+ }
+}
+
+// Start boots up the announcement based synchroniser, accepting and processing
+// hash notifications and block fetches until termination requested.
+func (f *TxFetcher) Start() {
+ go f.loop()
+}
+
+// Stop terminates the announcement based synchroniser, canceling all pending
+// operations.
+func (f *TxFetcher) Stop() {
+ close(f.quit)
+}
+
+func (f *TxFetcher) loop() {
+ var (
+ waitTimer = new(mclock.Timer)
+ timeoutTimer = new(mclock.Timer)
+
+ waitTrigger = make(chan struct{}, 1)
+ timeoutTrigger = make(chan struct{}, 1)
+ )
+ for {
+ select {
+ case ann := <-f.notify:
+ // Drop part of the new announcements if there are too many accumulated.
+ // Note, we could but do not filter already known transactions here as
+ // the probability of something arriving between this call and the pre-
+ // filter outside is essentially zero.
+ used := len(f.waitslots[ann.origin]) + len(f.announces[ann.origin])
+ if used >= maxTxAnnounces {
+ // This can happen if a set of transactions are requested but not
+ // all fulfilled, so the remainder are rescheduled without the cap
+ // check. Should be fine as the limit is in the thousands and the
+ // request size in the hundreds.
+ txAnnounceDOSMeter.Mark(int64(len(ann.hashes)))
+ break
+ }
+ want := used + len(ann.hashes)
+ if want > maxTxAnnounces {
+ txAnnounceDOSMeter.Mark(int64(want - maxTxAnnounces))
+ ann.hashes = ann.hashes[:want-maxTxAnnounces]
+ }
+ // All is well, schedule the remainder of the transactions
+ idleWait := len(f.waittime) == 0
+ _, oldPeer := f.announces[ann.origin]
+
+ for _, hash := range ann.hashes {
+ // If the transaction is already downloading, add it to the list
+ // of possible alternates (in case the current retrieval fails) and
+ // also account it for the peer.
+ if f.alternates[hash] != nil {
+ f.alternates[hash][ann.origin] = struct{}{}
+
+ // Stage 2 and 3 share the set of origins per tx
+ if announces := f.announces[ann.origin]; announces != nil {
+ announces[hash] = struct{}{}
+ } else {
+ f.announces[ann.origin] = map[common.Hash]struct{}{hash: struct{}{}}
+ }
+ continue
+ }
+ // If the transaction is not downloading, but is already queued
+ // from a different peer, track it for the new peer too.
+ if f.announced[hash] != nil {
+ f.announced[hash][ann.origin] = struct{}{}
+
+ // Stage 2 and 3 share the set of origins per tx
+ if announces := f.announces[ann.origin]; announces != nil {
+ announces[hash] = struct{}{}
+ } else {
+ f.announces[ann.origin] = map[common.Hash]struct{}{hash: struct{}{}}
+ }
+ continue
+ }
+ // If the transaction is already known to the fetcher, but not
+ // yet downloading, add the peer as an alternate origin in the
+ // waiting list.
+ if f.waitlist[hash] != nil {
+ f.waitlist[hash][ann.origin] = struct{}{}
+
+ if waitslots := f.waitslots[ann.origin]; waitslots != nil {
+ waitslots[hash] = struct{}{}
+ } else {
+ f.waitslots[ann.origin] = map[common.Hash]struct{}{hash: struct{}{}}
+ }
+ continue
+ }
+ // Transaction unknown to the fetcher, insert it into the waiting list
+ f.waitlist[hash] = map[string]struct{}{ann.origin: struct{}{}}
+ f.waittime[hash] = f.clock.Now()
+
+ if waitslots := f.waitslots[ann.origin]; waitslots != nil {
+ waitslots[hash] = struct{}{}
+ } else {
+ f.waitslots[ann.origin] = map[common.Hash]struct{}{hash: struct{}{}}
+ }
+ }
+ // If a new item was added to the waitlist, schedule it into the fetcher
+ if idleWait && len(f.waittime) > 0 {
+ f.rescheduleWait(waitTimer, waitTrigger)
+ }
+ // If this peer is new and announced something already queued, maybe
+ // request transactions from them
+ if !oldPeer && len(f.announces[ann.origin]) > 0 {
+ f.scheduleFetches(timeoutTimer, timeoutTrigger, map[string]struct{}{ann.origin: struct{}{}})
+ }
+
+ case <-waitTrigger:
+ // At least one transaction's waiting time ran out, push all expired
+ // ones into the retrieval queues
+ actives := make(map[string]struct{})
+ for hash, instance := range f.waittime {
+ if time.Duration(f.clock.Now()-instance)+txGatherSlack > txArriveTimeout {
+ // Transaction expired without propagation, schedule for retrieval
+ if f.announced[hash] != nil {
+ panic("announce tracker already contains waitlist item")
+ }
+ f.announced[hash] = f.waitlist[hash]
+ for peer := range f.waitlist[hash] {
+ if announces := f.announces[peer]; announces != nil {
+ announces[hash] = struct{}{}
+ } else {
+ f.announces[peer] = map[common.Hash]struct{}{hash: struct{}{}}
+ }
+ delete(f.waitslots[peer], hash)
+ if len(f.waitslots[peer]) == 0 {
+ delete(f.waitslots, peer)
+ }
+ actives[peer] = struct{}{}
+ }
+ delete(f.waittime, hash)
+ delete(f.waitlist, hash)
+ }
+ }
+ // If transactions are still waiting for propagation, reschedule the wait timer
+ if len(f.waittime) > 0 {
+ f.rescheduleWait(waitTimer, waitTrigger)
+ }
+ // If any peers became active and are idle, request transactions from them
+ if len(actives) > 0 {
+ f.scheduleFetches(timeoutTimer, timeoutTrigger, actives)
+ }
+
+ case <-timeoutTrigger:
+ // Clean up any expired retrievals and avoid re-requesting them from the
+ // same peer (either overloaded or malicious, useless in both cases). We
+ // could also penalize (Drop), but there's nothing to gain, and if could
+ // possibly further increase the load on it.
+ for peer, req := range f.requests {
+ if time.Duration(f.clock.Now()-req.time)+txGatherSlack > txFetchTimeout {
+ txRequestTimeoutMeter.Mark(int64(len(req.hashes)))
+
+ // Reschedule all the not-yet-delivered fetches to alternate peers
+ for _, hash := range req.hashes {
+ // Skip rescheduling hashes already delivered by someone else
+ if req.stolen != nil {
+ if _, ok := req.stolen[hash]; ok {
+ continue
+ }
+ }
+ // Move the delivery back from fetching to queued
+ if _, ok := f.announced[hash]; ok {
+ panic("announced tracker already contains alternate item")
+ }
+ if f.alternates[hash] != nil { // nil if tx was broadcast during fetch
+ f.announced[hash] = f.alternates[hash]
+ }
+ delete(f.announced[hash], peer)
+ if len(f.announced[hash]) == 0 {
+ delete(f.announced, hash)
+ }
+ delete(f.announces[peer], hash)
+ delete(f.alternates, hash)
+ delete(f.fetching, hash)
+ }
+ if len(f.announces[peer]) == 0 {
+ delete(f.announces, peer)
+ }
+ // Keep track of the request as dangling, but never expire
+ f.requests[peer].hashes = nil
+ }
+ }
+ // Schedule a new transaction retrieval
+ f.scheduleFetches(timeoutTimer, timeoutTrigger, nil)
+
+ // No idea if we sheduled something or not, trigger the timer if needed
+ // TODO(karalabe): this is kind of lame, can't we dump it into scheduleFetches somehow?
+ f.rescheduleTimeout(timeoutTimer, timeoutTrigger)
+
+ case delivery := <-f.cleanup:
+ // Independent if the delivery was direct or broadcast, remove all
+ // traces of the hash from internal trackers
+ for _, hash := range delivery.hashes {
+ if _, ok := f.waitlist[hash]; ok {
+ for peer, txset := range f.waitslots {
+ delete(txset, hash)
+ if len(txset) == 0 {
+ delete(f.waitslots, peer)
+ }
+ }
+ delete(f.waitlist, hash)
+ delete(f.waittime, hash)
+ } else {
+ for peer, txset := range f.announces {
+ delete(txset, hash)
+ if len(txset) == 0 {
+ delete(f.announces, peer)
+ }
+ }
+ delete(f.announced, hash)
+ delete(f.alternates, hash)
+
+ // If a transaction currently being fetched from a different
+ // origin was delivered (delivery stolen), mark it so the
+ // actual delivery won't double schedule it.
+ if origin, ok := f.fetching[hash]; ok && (origin != delivery.origin || !delivery.direct) {
+ stolen := f.requests[origin].stolen
+ if stolen == nil {
+ f.requests[origin].stolen = make(map[common.Hash]struct{})
+ stolen = f.requests[origin].stolen
+ }
+ stolen[hash] = struct{}{}
+ }
+ delete(f.fetching, hash)
+ }
+ }
+ // In case of a direct delivery, also reschedule anything missing
+ // from the original query
+ if delivery.direct {
+ // Mark the reqesting successful (independent of individual status)
+ txRequestDoneMeter.Mark(int64(len(delivery.hashes)))
+
+ // Make sure something was pending, nuke it
+ req := f.requests[delivery.origin]
+ if req == nil {
+ log.Warn("Unexpected transaction delivery", "peer", delivery.origin)
+ break
+ }
+ delete(f.requests, delivery.origin)
+
+ // Anything not delivered should be re-scheduled (with or without
+ // this peer, depending on the response cutoff)
+ delivered := make(map[common.Hash]struct{})
+ for _, hash := range delivery.hashes {
+ delivered[hash] = struct{}{}
+ }
+ cutoff := len(req.hashes) // If nothing is delivered, assume everything is missing, don't retry!!!
+ for i, hash := range req.hashes {
+ if _, ok := delivered[hash]; ok {
+ cutoff = i
+ }
+ }
+ // Reschedule missing hashes from alternates, not-fulfilled from alt+self
+ for i, hash := range req.hashes {
+ // Skip rescheduling hashes already delivered by someone else
+ if req.stolen != nil {
+ if _, ok := req.stolen[hash]; ok {
+ continue
+ }
+ }
+ if _, ok := delivered[hash]; !ok {
+ if i < cutoff {
+ delete(f.alternates[hash], delivery.origin)
+ delete(f.announces[delivery.origin], hash)
+ if len(f.announces[delivery.origin]) == 0 {
+ delete(f.announces, delivery.origin)
+ }
+ }
+ if len(f.alternates[hash]) > 0 {
+ if _, ok := f.announced[hash]; ok {
+ panic(fmt.Sprintf("announced tracker already contains alternate item: %v", f.announced[hash]))
+ }
+ f.announced[hash] = f.alternates[hash]
+ }
+ }
+ delete(f.alternates, hash)
+ delete(f.fetching, hash)
+ }
+ // Something was delivered, try to rechedule requests
+ f.scheduleFetches(timeoutTimer, timeoutTrigger, nil) // Partial delivery may enable others to deliver too
+ }
+
+ case drop := <-f.drop:
+ // A peer was dropped, remove all traces of it
+ if _, ok := f.waitslots[drop.peer]; ok {
+ for hash := range f.waitslots[drop.peer] {
+ delete(f.waitlist[hash], drop.peer)
+ if len(f.waitlist[hash]) == 0 {
+ delete(f.waitlist, hash)
+ delete(f.waittime, hash)
+ }
+ }
+ delete(f.waitslots, drop.peer)
+ if len(f.waitlist) > 0 {
+ f.rescheduleWait(waitTimer, waitTrigger)
+ }
+ }
+ // Clean up any active requests
+ var request *txRequest
+ if request = f.requests[drop.peer]; request != nil {
+ for _, hash := range request.hashes {
+ // Skip rescheduling hashes already delivered by someone else
+ if request.stolen != nil {
+ if _, ok := request.stolen[hash]; ok {
+ continue
+ }
+ }
+ // Undelivered hash, reschedule if there's an alternative origin available
+ delete(f.alternates[hash], drop.peer)
+ if len(f.alternates[hash]) == 0 {
+ delete(f.alternates, hash)
+ } else {
+ f.announced[hash] = f.alternates[hash]
+ delete(f.alternates, hash)
+ }
+ delete(f.fetching, hash)
+ }
+ delete(f.requests, drop.peer)
+ }
+ // Clean up general announcement tracking
+ if _, ok := f.announces[drop.peer]; ok {
+ for hash := range f.announces[drop.peer] {
+ delete(f.announced[hash], drop.peer)
+ if len(f.announced[hash]) == 0 {
+ delete(f.announced, hash)
+ }
+ }
+ delete(f.announces, drop.peer)
+ }
+ // If a request was cancelled, check if anything needs to be rescheduled
+ if request != nil {
+ f.scheduleFetches(timeoutTimer, timeoutTrigger, nil)
+ f.rescheduleTimeout(timeoutTimer, timeoutTrigger)
+ }
+
+ case <-f.quit:
+ return
+ }
+ // No idea what happened, but bump some sanity metrics
+ txFetcherWaitingPeers.Update(int64(len(f.waitslots)))
+ txFetcherWaitingHashes.Update(int64(len(f.waitlist)))
+ txFetcherQueueingPeers.Update(int64(len(f.announces) - len(f.requests)))
+ txFetcherQueueingHashes.Update(int64(len(f.announced)))
+ txFetcherFetchingPeers.Update(int64(len(f.requests)))
+ txFetcherFetchingHashes.Update(int64(len(f.fetching)))
+
+ // Loop did something, ping the step notifier if needed (tests)
+ if f.step != nil {
+ f.step <- struct{}{}
+ }
+ }
+}
+
+// rescheduleWait iterates over all the transactions currently in the waitlist
+// and schedules the movement into the fetcher for the earliest.
+//
+// The method has a granularity of 'gatherSlack', since there's not much point in
+// spinning over all the transactions just to maybe find one that should trigger
+// a few ms earlier.
+func (f *TxFetcher) rescheduleWait(timer *mclock.Timer, trigger chan struct{}) {
+ if *timer != nil {
+ (*timer).Stop()
+ }
+ now := f.clock.Now()
+
+ earliest := now
+ for _, instance := range f.waittime {
+ if earliest > instance {
+ earliest = instance
+ if txArriveTimeout-time.Duration(now-earliest) < gatherSlack {
+ break
+ }
+ }
+ }
+ *timer = f.clock.AfterFunc(txArriveTimeout-time.Duration(now-earliest), func() {
+ trigger <- struct{}{}
+ })
+}
+
+// rescheduleTimeout iterates over all the transactions currently in flight and
+// schedules a cleanup run when the first would trigger.
+//
+// The method has a granularity of 'gatherSlack', since there's not much point in
+// spinning over all the transactions just to maybe find one that should trigger
+// a few ms earlier.
+//
+// This method is a bit "flaky" "by design". In theory the timeout timer only ever
+// should be rescheduled if some request is pending. In practice, a timeout will
+// cause the timer to be rescheduled every 5 secs (until the peer comes through or
+// disconnects). This is a limitation of the fetcher code because we don't trac
+// pending requests and timed out requests separatey. Without double tracking, if
+// we simply didn't reschedule the timer on all-timeout then the timer would never
+// be set again since len(request) > 0 => something's running.
+func (f *TxFetcher) rescheduleTimeout(timer *mclock.Timer, trigger chan struct{}) {
+ if *timer != nil {
+ (*timer).Stop()
+ }
+ now := f.clock.Now()
+
+ earliest := now
+ for _, req := range f.requests {
+ // If this request already timed out, skip it altogether
+ if req.hashes == nil {
+ continue
+ }
+ if earliest > req.time {
+ earliest = req.time
+ if txFetchTimeout-time.Duration(now-earliest) < gatherSlack {
+ break
+ }
+ }
+ }
+ *timer = f.clock.AfterFunc(txFetchTimeout-time.Duration(now-earliest), func() {
+ trigger <- struct{}{}
+ })
+}
+
+// scheduleFetches starts a batch of retrievals for all available idle peers.
+func (f *TxFetcher) scheduleFetches(timer *mclock.Timer, timeout chan struct{}, whitelist map[string]struct{}) {
+ // Gather the set of peers we want to retrieve from (default to all)
+ actives := whitelist
+ if actives == nil {
+ actives = make(map[string]struct{})
+ for peer := range f.announces {
+ actives[peer] = struct{}{}
+ }
+ }
+ if len(actives) == 0 {
+ return
+ }
+ // For each active peer, try to schedule some transaction fetches
+ idle := len(f.requests) == 0
+
+ f.forEachPeer(actives, func(peer string) {
+ if f.requests[peer] != nil {
+ return // continue in the for-each
+ }
+ if len(f.announces[peer]) == 0 {
+ return // continue in the for-each
+ }
+ hashes := make([]common.Hash, 0, maxTxRetrievals)
+ f.forEachHash(f.announces[peer], func(hash common.Hash) bool {
+ if _, ok := f.fetching[hash]; !ok {
+ // Mark the hash as fetching and stash away possible alternates
+ f.fetching[hash] = peer
+
+ if _, ok := f.alternates[hash]; ok {
+ panic(fmt.Sprintf("alternate tracker already contains fetching item: %v", f.alternates[hash]))
+ }
+ f.alternates[hash] = f.announced[hash]
+ delete(f.announced, hash)
+
+ // Accumulate the hash and stop if the limit was reached
+ hashes = append(hashes, hash)
+ if len(hashes) >= maxTxRetrievals {
+ return false // break in the for-each
+ }
+ }
+ return true // continue in the for-each
+ })
+ // If any hashes were allocated, request them from the peer
+ if len(hashes) > 0 {
+ f.requests[peer] = &txRequest{hashes: hashes, time: f.clock.Now()}
+ txRequestOutMeter.Mark(int64(len(hashes)))
+
+ go func(peer string, hashes []common.Hash) {
+ // Try to fetch the transactions, but in case of a request
+ // failure (e.g. peer disconnected), reschedule the hashes.
+ if err := f.fetchTxs(peer, hashes); err != nil {
+ txRequestFailMeter.Mark(int64(len(hashes)))
+ f.Drop(peer)
+ }
+ }(peer, hashes)
+ }
+ })
+ // If a new request was fired, schedule a timeout timer
+ if idle && len(f.requests) > 0 {
+ f.rescheduleTimeout(timer, timeout)
+ }
+}
+
+// forEachPeer does a range loop over a map of peers in production, but during
+// testing it does a deterministic sorted random to allow reproducing issues.
+func (f *TxFetcher) forEachPeer(peers map[string]struct{}, do func(peer string)) {
+ // If we're running production, use whatever Go's map gives us
+ if f.rand == nil {
+ for peer := range peers {
+ do(peer)
+ }
+ return
+ }
+ // We're running the test suite, make iteration deterministic
+ list := make([]string, 0, len(peers))
+ for peer := range peers {
+ list = append(list, peer)
+ }
+ sort.Strings(list)
+ rotateStrings(list, f.rand.Intn(len(list)))
+ for _, peer := range list {
+ do(peer)
+ }
+}
+
+// forEachHash does a range loop over a map of hashes in production, but during
+// testing it does a deterministic sorted random to allow reproducing issues.
+func (f *TxFetcher) forEachHash(hashes map[common.Hash]struct{}, do func(hash common.Hash) bool) {
+ // If we're running production, use whatever Go's map gives us
+ if f.rand == nil {
+ for hash := range hashes {
+ if !do(hash) {
+ return
+ }
+ }
+ return
+ }
+ // We're running the test suite, make iteration deterministic
+ list := make([]common.Hash, 0, len(hashes))
+ for hash := range hashes {
+ list = append(list, hash)
+ }
+ sortHashes(list)
+ rotateHashes(list, f.rand.Intn(len(list)))
+ for _, hash := range list {
+ if !do(hash) {
+ return
+ }
+ }
+}
+
+// rotateStrings rotates the contents of a slice by n steps. This method is only
+// used in tests to simulate random map iteration but keep it deterministic.
+func rotateStrings(slice []string, n int) {
+ orig := make([]string, len(slice))
+ copy(orig, slice)
+
+ for i := 0; i < len(orig); i++ {
+ slice[i] = orig[(i+n)%len(orig)]
+ }
+}
+
+// sortHashes sorts a slice of hashes. This method is only used in tests in order
+// to simulate random map iteration but keep it deterministic.
+func sortHashes(slice []common.Hash) {
+ for i := 0; i < len(slice); i++ {
+ for j := i + 1; j < len(slice); j++ {
+ if bytes.Compare(slice[i][:], slice[j][:]) > 0 {
+ slice[i], slice[j] = slice[j], slice[i]
+ }
+ }
+ }
+}
+
+// rotateHashes rotates the contents of a slice by n steps. This method is only
+// used in tests to simulate random map iteration but keep it deterministic.
+func rotateHashes(slice []common.Hash, n int) {
+ orig := make([]common.Hash, len(slice))
+ copy(orig, slice)
+
+ for i := 0; i < len(orig); i++ {
+ slice[i] = orig[(i+n)%len(orig)]
+ }
+}
diff --git a/eth/fetcher/tx_fetcher_test.go b/eth/fetcher/tx_fetcher_test.go
new file mode 100644
index 000000000000..4831fadbd439
--- /dev/null
+++ b/eth/fetcher/tx_fetcher_test.go
@@ -0,0 +1,1528 @@
+// Copyright 2020 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package fetcher
+
+import (
+ "errors"
+ "math/big"
+ "math/rand"
+ "testing"
+ "time"
+
+ "github.com/XinFinOrg/XDPoSChain/common"
+ "github.com/XinFinOrg/XDPoSChain/common/mclock"
+ "github.com/XinFinOrg/XDPoSChain/core"
+ "github.com/XinFinOrg/XDPoSChain/core/types"
+)
+
+var (
+ // testTxs is a set of transactions to use during testing that have meaninful hashes.
+ testTxs = []*types.Transaction{
+ types.NewTransaction(rand.Uint64(), common.Address{byte(rand.Intn(256))}, new(big.Int), 0, new(big.Int), nil),
+ types.NewTransaction(rand.Uint64(), common.Address{byte(rand.Intn(256))}, new(big.Int), 0, new(big.Int), nil),
+ types.NewTransaction(rand.Uint64(), common.Address{byte(rand.Intn(256))}, new(big.Int), 0, new(big.Int), nil),
+ types.NewTransaction(rand.Uint64(), common.Address{byte(rand.Intn(256))}, new(big.Int), 0, new(big.Int), nil),
+ }
+ // testTxsHashes is the hashes of the test transactions above
+ testTxsHashes = []common.Hash{testTxs[0].Hash(), testTxs[1].Hash(), testTxs[2].Hash(), testTxs[3].Hash()}
+)
+
+type doTxNotify struct {
+ peer string
+ hashes []common.Hash
+}
+type doTxEnqueue struct {
+ peer string
+ txs []*types.Transaction
+ direct bool
+}
+type doWait struct {
+ time time.Duration
+ step bool
+}
+type doDrop string
+type doFunc func()
+
+type isWaiting map[string][]common.Hash
+type isScheduled struct {
+ tracking map[string][]common.Hash
+ fetching map[string][]common.Hash
+ dangling map[string][]common.Hash
+}
+type isUnderpriced int
+
+// txFetcherTest represents a test scenario that can be executed by the test
+// runner.
+type txFetcherTest struct {
+ init func() *TxFetcher
+ steps []interface{}
+}
+
+// Tests that transaction announcements are added to a waitlist, and none
+// of them are scheduled for retrieval until the wait expires.
+func TestTransactionFetcherWaiting(t *testing.T) {
+ testTransactionFetcherParallel(t, txFetcherTest{
+ init: func() *TxFetcher {
+ return NewTxFetcher(
+ func(common.Hash) bool { return false },
+ nil,
+ func(string, []common.Hash) error { return nil },
+ )
+ },
+ steps: []interface{}{
+ // Initial announcement to get something into the waitlist
+ doTxNotify{peer: "A", hashes: []common.Hash{{0x01}, {0x02}}},
+ isWaiting(map[string][]common.Hash{
+ "A": {{0x01}, {0x02}},
+ }),
+ // Announce from a new peer to check that no overwrite happens
+ doTxNotify{peer: "B", hashes: []common.Hash{{0x03}, {0x04}}},
+ isWaiting(map[string][]common.Hash{
+ "A": {{0x01}, {0x02}},
+ "B": {{0x03}, {0x04}},
+ }),
+ // Announce clashing hashes but unique new peer
+ doTxNotify{peer: "C", hashes: []common.Hash{{0x01}, {0x04}}},
+ isWaiting(map[string][]common.Hash{
+ "A": {{0x01}, {0x02}},
+ "B": {{0x03}, {0x04}},
+ "C": {{0x01}, {0x04}},
+ }),
+ // Announce existing and clashing hashes from existing peer
+ doTxNotify{peer: "A", hashes: []common.Hash{{0x01}, {0x03}, {0x05}}},
+ isWaiting(map[string][]common.Hash{
+ "A": {{0x01}, {0x02}, {0x03}, {0x05}},
+ "B": {{0x03}, {0x04}},
+ "C": {{0x01}, {0x04}},
+ }),
+ isScheduled{tracking: nil, fetching: nil},
+
+ // Wait for the arrival timeout which should move all expired items
+ // from the wait list to the scheduler
+ doWait{time: txArriveTimeout, step: true},
+ isWaiting(nil),
+ isScheduled{
+ tracking: map[string][]common.Hash{
+ "A": {{0x01}, {0x02}, {0x03}, {0x05}},
+ "B": {{0x03}, {0x04}},
+ "C": {{0x01}, {0x04}},
+ },
+ fetching: map[string][]common.Hash{ // Depends on deterministic test randomizer
+ "A": {{0x02}, {0x03}, {0x05}},
+ "C": {{0x01}, {0x04}},
+ },
+ },
+ // Queue up a non-fetchable transaction and then trigger it with a new
+ // peer (weird case to test 1 line in the fetcher)
+ doTxNotify{peer: "C", hashes: []common.Hash{{0x06}, {0x07}}},
+ isWaiting(map[string][]common.Hash{
+ "C": {{0x06}, {0x07}},
+ }),
+ doWait{time: txArriveTimeout, step: true},
+ isScheduled{
+ tracking: map[string][]common.Hash{
+ "A": {{0x01}, {0x02}, {0x03}, {0x05}},
+ "B": {{0x03}, {0x04}},
+ "C": {{0x01}, {0x04}, {0x06}, {0x07}},
+ },
+ fetching: map[string][]common.Hash{
+ "A": {{0x02}, {0x03}, {0x05}},
+ "C": {{0x01}, {0x04}},
+ },
+ },
+ doTxNotify{peer: "D", hashes: []common.Hash{{0x06}, {0x07}}},
+ isScheduled{
+ tracking: map[string][]common.Hash{
+ "A": {{0x01}, {0x02}, {0x03}, {0x05}},
+ "B": {{0x03}, {0x04}},
+ "C": {{0x01}, {0x04}, {0x06}, {0x07}},
+ "D": {{0x06}, {0x07}},
+ },
+ fetching: map[string][]common.Hash{
+ "A": {{0x02}, {0x03}, {0x05}},
+ "C": {{0x01}, {0x04}},
+ "D": {{0x06}, {0x07}},
+ },
+ },
+ },
+ })
+}
+
+// Tests that transaction announcements skip the waiting list if they are
+// already scheduled.
+func TestTransactionFetcherSkipWaiting(t *testing.T) {
+ testTransactionFetcherParallel(t, txFetcherTest{
+ init: func() *TxFetcher {
+ return NewTxFetcher(
+ func(common.Hash) bool { return false },
+ nil,
+ func(string, []common.Hash) error { return nil },
+ )
+ },
+ steps: []interface{}{
+ // Push an initial announcement through to the scheduled stage
+ doTxNotify{peer: "A", hashes: []common.Hash{{0x01}, {0x02}}},
+ isWaiting(map[string][]common.Hash{
+ "A": {{0x01}, {0x02}},
+ }),
+ isScheduled{tracking: nil, fetching: nil},
+
+ doWait{time: txArriveTimeout, step: true},
+ isWaiting(nil),
+ isScheduled{
+ tracking: map[string][]common.Hash{
+ "A": {{0x01}, {0x02}},
+ },
+ fetching: map[string][]common.Hash{
+ "A": {{0x01}, {0x02}},
+ },
+ },
+ // Announce overlaps from the same peer, ensure the new ones end up
+ // in stage one, and clashing ones don't get double tracked
+ doTxNotify{peer: "A", hashes: []common.Hash{{0x02}, {0x03}}},
+ isWaiting(map[string][]common.Hash{
+ "A": {{0x03}},
+ }),
+ isScheduled{
+ tracking: map[string][]common.Hash{
+ "A": {{0x01}, {0x02}},
+ },
+ fetching: map[string][]common.Hash{
+ "A": {{0x01}, {0x02}},
+ },
+ },
+ // Announce overlaps from a new peer, ensure new transactions end up
+ // in stage one and clashing ones get tracked for the new peer
+ doTxNotify{peer: "B", hashes: []common.Hash{{0x02}, {0x03}, {0x04}}},
+ isWaiting(map[string][]common.Hash{
+ "A": {{0x03}},
+ "B": {{0x03}, {0x04}},
+ }),
+ isScheduled{
+ tracking: map[string][]common.Hash{
+ "A": {{0x01}, {0x02}},
+ "B": {{0x02}},
+ },
+ fetching: map[string][]common.Hash{
+ "A": {{0x01}, {0x02}},
+ },
+ },
+ },
+ })
+}
+
+// Tests that only a single transaction request gets scheduled to a peer
+// and subsequent announces block or get allotted to someone else.
+func TestTransactionFetcherSingletonRequesting(t *testing.T) {
+ testTransactionFetcherParallel(t, txFetcherTest{
+ init: func() *TxFetcher {
+ return NewTxFetcher(
+ func(common.Hash) bool { return false },
+ nil,
+ func(string, []common.Hash) error { return nil },
+ )
+ },
+ steps: []interface{}{
+ // Push an initial announcement through to the scheduled stage
+ doTxNotify{peer: "A", hashes: []common.Hash{{0x01}, {0x02}}},
+ isWaiting(map[string][]common.Hash{
+ "A": {{0x01}, {0x02}},
+ }),
+ isScheduled{tracking: nil, fetching: nil},
+
+ doWait{time: txArriveTimeout, step: true},
+ isWaiting(nil),
+ isScheduled{
+ tracking: map[string][]common.Hash{
+ "A": {{0x01}, {0x02}},
+ },
+ fetching: map[string][]common.Hash{
+ "A": {{0x01}, {0x02}},
+ },
+ },
+ // Announce a new set of transactions from the same peer and ensure
+ // they do not start fetching since the peer is already busy
+ doTxNotify{peer: "A", hashes: []common.Hash{{0x03}, {0x04}}},
+ isWaiting(map[string][]common.Hash{
+ "A": {{0x03}, {0x04}},
+ }),
+ isScheduled{
+ tracking: map[string][]common.Hash{
+ "A": {{0x01}, {0x02}},
+ },
+ fetching: map[string][]common.Hash{
+ "A": {{0x01}, {0x02}},
+ },
+ },
+ doWait{time: txArriveTimeout, step: true},
+ isWaiting(nil),
+ isScheduled{
+ tracking: map[string][]common.Hash{
+ "A": {{0x01}, {0x02}, {0x03}, {0x04}},
+ },
+ fetching: map[string][]common.Hash{
+ "A": {{0x01}, {0x02}},
+ },
+ },
+ // Announce a duplicate set of transactions from a new peer and ensure
+ // uniquely new ones start downloading, even if clashing.
+ doTxNotify{peer: "B", hashes: []common.Hash{{0x02}, {0x03}, {0x05}, {0x06}}},
+ isWaiting(map[string][]common.Hash{
+ "B": {{0x05}, {0x06}},
+ }),
+ isScheduled{
+ tracking: map[string][]common.Hash{
+ "A": {{0x01}, {0x02}, {0x03}, {0x04}},
+ "B": {{0x02}, {0x03}},
+ },
+ fetching: map[string][]common.Hash{
+ "A": {{0x01}, {0x02}},
+ "B": {{0x03}},
+ },
+ },
+ },
+ })
+}
+
+// Tests that if a transaction retrieval fails, all the transactions get
+// instantly schedule back to someone else or the announcements dropped
+// if no alternate source is available.
+func TestTransactionFetcherFailedRescheduling(t *testing.T) {
+ // Create a channel to control when tx requests can fail
+ proceed := make(chan struct{})
+
+ testTransactionFetcherParallel(t, txFetcherTest{
+ init: func() *TxFetcher {
+ return NewTxFetcher(
+ func(common.Hash) bool { return false },
+ nil,
+ func(origin string, hashes []common.Hash) error {
+ <-proceed
+ return errors.New("peer disconnected")
+ },
+ )
+ },
+ steps: []interface{}{
+ // Push an initial announcement through to the scheduled stage
+ doTxNotify{peer: "A", hashes: []common.Hash{{0x01}, {0x02}}},
+ isWaiting(map[string][]common.Hash{
+ "A": {{0x01}, {0x02}},
+ }),
+ isScheduled{tracking: nil, fetching: nil},
+
+ doWait{time: txArriveTimeout, step: true},
+ isWaiting(nil),
+ isScheduled{
+ tracking: map[string][]common.Hash{
+ "A": {{0x01}, {0x02}},
+ },
+ fetching: map[string][]common.Hash{
+ "A": {{0x01}, {0x02}},
+ },
+ },
+ // While the original peer is stuck in the request, push in an second
+ // data source.
+ doTxNotify{peer: "B", hashes: []common.Hash{{0x02}}},
+ isWaiting(nil),
+ isScheduled{
+ tracking: map[string][]common.Hash{
+ "A": {{0x01}, {0x02}},
+ "B": {{0x02}},
+ },
+ fetching: map[string][]common.Hash{
+ "A": {{0x01}, {0x02}},
+ },
+ },
+ // Wait until the original request fails and check that transactions
+ // are either rescheduled or dropped
+ doFunc(func() {
+ proceed <- struct{}{} // Allow peer A to return the failure
+ }),
+ doWait{time: 0, step: true},
+ isWaiting(nil),
+ isScheduled{
+ tracking: map[string][]common.Hash{
+ "B": {{0x02}},
+ },
+ fetching: map[string][]common.Hash{
+ "B": {{0x02}},
+ },
+ },
+ doFunc(func() {
+ proceed <- struct{}{} // Allow peer B to return the failure
+ }),
+ doWait{time: 0, step: true},
+ isWaiting(nil),
+ isScheduled{nil, nil, nil},
+ },
+ })
+}
+
+// Tests that if a transaction retrieval succeeds, all alternate origins
+// are cleaned up.
+func TestTransactionFetcherCleanup(t *testing.T) {
+ testTransactionFetcherParallel(t, txFetcherTest{
+ init: func() *TxFetcher {
+ return NewTxFetcher(
+ func(common.Hash) bool { return false },
+ func(txs []*types.Transaction) []error {
+ return make([]error, len(txs))
+ },
+ func(string, []common.Hash) error { return nil },
+ )
+ },
+ steps: []interface{}{
+ // Push an initial announcement through to the scheduled stage
+ doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}},
+ isWaiting(map[string][]common.Hash{
+ "A": {testTxsHashes[0]},
+ }),
+ isScheduled{tracking: nil, fetching: nil},
+
+ doWait{time: txArriveTimeout, step: true},
+ isWaiting(nil),
+ isScheduled{
+ tracking: map[string][]common.Hash{
+ "A": {testTxsHashes[0]},
+ },
+ fetching: map[string][]common.Hash{
+ "A": {testTxsHashes[0]},
+ },
+ },
+ // Request should be delivered
+ doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[0]}, direct: true},
+ isScheduled{nil, nil, nil},
+ },
+ })
+}
+
+// Tests that if a transaction retrieval succeeds, but the response is empty (no
+// transactions available, then all are nuked instead of being rescheduled (yes,
+// this was a bug)).
+func TestTransactionFetcherCleanupEmpty(t *testing.T) {
+ testTransactionFetcherParallel(t, txFetcherTest{
+ init: func() *TxFetcher {
+ return NewTxFetcher(
+ func(common.Hash) bool { return false },
+ func(txs []*types.Transaction) []error {
+ return make([]error, len(txs))
+ },
+ func(string, []common.Hash) error { return nil },
+ )
+ },
+ steps: []interface{}{
+ // Push an initial announcement through to the scheduled stage
+ doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}},
+ isWaiting(map[string][]common.Hash{
+ "A": {testTxsHashes[0]},
+ }),
+ isScheduled{tracking: nil, fetching: nil},
+
+ doWait{time: txArriveTimeout, step: true},
+ isWaiting(nil),
+ isScheduled{
+ tracking: map[string][]common.Hash{
+ "A": {testTxsHashes[0]},
+ },
+ fetching: map[string][]common.Hash{
+ "A": {testTxsHashes[0]},
+ },
+ },
+ // Deliver an empty response and ensure the transaction is cleared, not rescheduled
+ doTxEnqueue{peer: "A", txs: []*types.Transaction{}, direct: true},
+ isScheduled{nil, nil, nil},
+ },
+ })
+}
+
+// Tests that non-returned transactions are either re-sheduled from a
+// different peer, or self if they are after the cutoff point.
+func TestTransactionFetcherMissingRescheduling(t *testing.T) {
+ testTransactionFetcherParallel(t, txFetcherTest{
+ init: func() *TxFetcher {
+ return NewTxFetcher(
+ func(common.Hash) bool { return false },
+ func(txs []*types.Transaction) []error {
+ return make([]error, len(txs))
+ },
+ func(string, []common.Hash) error { return nil },
+ )
+ },
+ steps: []interface{}{
+ // Push an initial announcement through to the scheduled stage
+ doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0], testTxsHashes[1], testTxsHashes[2]}},
+ isWaiting(map[string][]common.Hash{
+ "A": {testTxsHashes[0], testTxsHashes[1], testTxsHashes[2]},
+ }),
+ isScheduled{tracking: nil, fetching: nil},
+
+ doWait{time: txArriveTimeout, step: true},
+ isWaiting(nil),
+ isScheduled{
+ tracking: map[string][]common.Hash{
+ "A": {testTxsHashes[0], testTxsHashes[1], testTxsHashes[2]},
+ },
+ fetching: map[string][]common.Hash{
+ "A": {testTxsHashes[0], testTxsHashes[1], testTxsHashes[2]},
+ },
+ },
+ // Deliver the middle transaction requested, the one before which
+ // should be dropped and the one after re-requested.
+ doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[0]}, direct: true}, // This depends on the deterministic random
+ isScheduled{
+ tracking: map[string][]common.Hash{
+ "A": {testTxsHashes[2]},
+ },
+ fetching: map[string][]common.Hash{
+ "A": {testTxsHashes[2]},
+ },
+ },
+ },
+ })
+}
+
+// Tests that out of two transactions, if one is missing and the last is
+// delivered, the peer gets properly cleaned out from the internal state.
+func TestTransactionFetcherMissingCleanup(t *testing.T) {
+ testTransactionFetcherParallel(t, txFetcherTest{
+ init: func() *TxFetcher {
+ return NewTxFetcher(
+ func(common.Hash) bool { return false },
+ func(txs []*types.Transaction) []error {
+ return make([]error, len(txs))
+ },
+ func(string, []common.Hash) error { return nil },
+ )
+ },
+ steps: []interface{}{
+ // Push an initial announcement through to the scheduled stage
+ doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0], testTxsHashes[1]}},
+ isWaiting(map[string][]common.Hash{
+ "A": {testTxsHashes[0], testTxsHashes[1]},
+ }),
+ isScheduled{tracking: nil, fetching: nil},
+
+ doWait{time: txArriveTimeout, step: true},
+ isWaiting(nil),
+ isScheduled{
+ tracking: map[string][]common.Hash{
+ "A": {testTxsHashes[0], testTxsHashes[1]},
+ },
+ fetching: map[string][]common.Hash{
+ "A": {testTxsHashes[0], testTxsHashes[1]},
+ },
+ },
+ // Deliver the middle transaction requested, the one before which
+ // should be dropped and the one after re-requested.
+ doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[1]}, direct: true}, // This depends on the deterministic random
+ isScheduled{nil, nil, nil},
+ },
+ })
+}
+
+// Tests that transaction broadcasts properly clean up announcements.
+func TestTransactionFetcherBroadcasts(t *testing.T) {
+ testTransactionFetcherParallel(t, txFetcherTest{
+ init: func() *TxFetcher {
+ return NewTxFetcher(
+ func(common.Hash) bool { return false },
+ func(txs []*types.Transaction) []error {
+ return make([]error, len(txs))
+ },
+ func(string, []common.Hash) error { return nil },
+ )
+ },
+ steps: []interface{}{
+ // Set up three transactions to be in different stats, waiting, queued and fetching
+ doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}},
+ doWait{time: txArriveTimeout, step: true},
+ doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[1]}},
+ doWait{time: txArriveTimeout, step: true},
+ doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[2]}},
+
+ isWaiting(map[string][]common.Hash{
+ "A": {testTxsHashes[2]},
+ }),
+ isScheduled{
+ tracking: map[string][]common.Hash{
+ "A": {testTxsHashes[0], testTxsHashes[1]},
+ },
+ fetching: map[string][]common.Hash{
+ "A": {testTxsHashes[0]},
+ },
+ },
+ // Broadcast all the transactions and ensure everything gets cleaned
+ // up, but the dangling request is left alone to avoid doing multiple
+ // concurrent requests.
+ doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[0], testTxs[1], testTxs[2]}, direct: false},
+ isWaiting(nil),
+ isScheduled{
+ tracking: nil,
+ fetching: nil,
+ dangling: map[string][]common.Hash{
+ "A": {testTxsHashes[0]},
+ },
+ },
+ // Deliver the requested hashes
+ doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[0], testTxs[1], testTxs[2]}, direct: true},
+ isScheduled{nil, nil, nil},
+ },
+ })
+}
+
+// Tests that the waiting list timers properly reset and reschedule.
+func TestTransactionFetcherWaitTimerResets(t *testing.T) {
+ testTransactionFetcherParallel(t, txFetcherTest{
+ init: func() *TxFetcher {
+ return NewTxFetcher(
+ func(common.Hash) bool { return false },
+ nil,
+ func(string, []common.Hash) error { return nil },
+ )
+ },
+ steps: []interface{}{
+ doTxNotify{peer: "A", hashes: []common.Hash{{0x01}}},
+ isWaiting(map[string][]common.Hash{
+ "A": {{0x01}},
+ }),
+ isScheduled{nil, nil, nil},
+ doWait{time: txArriveTimeout / 2, step: false},
+ isWaiting(map[string][]common.Hash{
+ "A": {{0x01}},
+ }),
+ isScheduled{nil, nil, nil},
+
+ doTxNotify{peer: "A", hashes: []common.Hash{{0x02}}},
+ isWaiting(map[string][]common.Hash{
+ "A": {{0x01}, {0x02}},
+ }),
+ isScheduled{nil, nil, nil},
+ doWait{time: txArriveTimeout / 2, step: true},
+ isWaiting(map[string][]common.Hash{
+ "A": {{0x02}},
+ }),
+ isScheduled{
+ tracking: map[string][]common.Hash{
+ "A": {{0x01}},
+ },
+ fetching: map[string][]common.Hash{
+ "A": {{0x01}},
+ },
+ },
+
+ doWait{time: txArriveTimeout / 2, step: true},
+ isWaiting(nil),
+ isScheduled{
+ tracking: map[string][]common.Hash{
+ "A": {{0x01}, {0x02}},
+ },
+ fetching: map[string][]common.Hash{
+ "A": {{0x01}},
+ },
+ },
+ },
+ })
+}
+
+// Tests that if a transaction request is not replied to, it will time
+// out and be re-scheduled for someone else.
+func TestTransactionFetcherTimeoutRescheduling(t *testing.T) {
+ testTransactionFetcherParallel(t, txFetcherTest{
+ init: func() *TxFetcher {
+ return NewTxFetcher(
+ func(common.Hash) bool { return false },
+ func(txs []*types.Transaction) []error {
+ return make([]error, len(txs))
+ },
+ func(string, []common.Hash) error { return nil },
+ )
+ },
+ steps: []interface{}{
+ // Push an initial announcement through to the scheduled stage
+ doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}},
+ isWaiting(map[string][]common.Hash{
+ "A": {testTxsHashes[0]},
+ }),
+ isScheduled{tracking: nil, fetching: nil},
+
+ doWait{time: txArriveTimeout, step: true},
+ isWaiting(nil),
+ isScheduled{
+ tracking: map[string][]common.Hash{
+ "A": {testTxsHashes[0]},
+ },
+ fetching: map[string][]common.Hash{
+ "A": {testTxsHashes[0]},
+ },
+ },
+ // Wait until the delivery times out, everything should be cleaned up
+ doWait{time: txFetchTimeout, step: true},
+ isWaiting(nil),
+ isScheduled{
+ tracking: nil,
+ fetching: nil,
+ dangling: map[string][]common.Hash{
+ "A": {},
+ },
+ },
+ // Ensure that followup announcements don't get scheduled
+ doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[1]}},
+ doWait{time: txArriveTimeout, step: true},
+ isScheduled{
+ tracking: map[string][]common.Hash{
+ "A": {testTxsHashes[1]},
+ },
+ fetching: nil,
+ dangling: map[string][]common.Hash{
+ "A": {},
+ },
+ },
+ // If the dangling request arrives a bit later, do not choke
+ doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[0]}, direct: true},
+ isWaiting(nil),
+ isScheduled{
+ tracking: map[string][]common.Hash{
+ "A": {testTxsHashes[1]},
+ },
+ fetching: map[string][]common.Hash{
+ "A": {testTxsHashes[1]},
+ },
+ },
+ },
+ })
+}
+
+// Tests that the fetching timeout timers properly reset and reschedule.
+func TestTransactionFetcherTimeoutTimerResets(t *testing.T) {
+ testTransactionFetcherParallel(t, txFetcherTest{
+ init: func() *TxFetcher {
+ return NewTxFetcher(
+ func(common.Hash) bool { return false },
+ nil,
+ func(string, []common.Hash) error { return nil },
+ )
+ },
+ steps: []interface{}{
+ doTxNotify{peer: "A", hashes: []common.Hash{{0x01}}},
+ doWait{time: txArriveTimeout, step: true},
+ doTxNotify{peer: "B", hashes: []common.Hash{{0x02}}},
+ doWait{time: txArriveTimeout, step: true},
+
+ isWaiting(nil),
+ isScheduled{
+ tracking: map[string][]common.Hash{
+ "A": {{0x01}},
+ "B": {{0x02}},
+ },
+ fetching: map[string][]common.Hash{
+ "A": {{0x01}},
+ "B": {{0x02}},
+ },
+ },
+ doWait{time: txFetchTimeout - txArriveTimeout, step: true},
+ isScheduled{
+ tracking: map[string][]common.Hash{
+ "B": {{0x02}},
+ },
+ fetching: map[string][]common.Hash{
+ "B": {{0x02}},
+ },
+ dangling: map[string][]common.Hash{
+ "A": {},
+ },
+ },
+ doWait{time: txArriveTimeout, step: true},
+ isScheduled{
+ tracking: nil,
+ fetching: nil,
+ dangling: map[string][]common.Hash{
+ "A": {},
+ "B": {},
+ },
+ },
+ },
+ })
+}
+
+// Tests that if thousands of transactions are announces, only a small
+// number of them will be requested at a time.
+func TestTransactionFetcherRateLimiting(t *testing.T) {
+ // Create a slew of transactions and to announce them
+ var hashes []common.Hash
+ for i := 0; i < maxTxAnnounces; i++ {
+ hashes = append(hashes, common.Hash{byte(i / 256), byte(i % 256)})
+ }
+
+ testTransactionFetcherParallel(t, txFetcherTest{
+ init: func() *TxFetcher {
+ return NewTxFetcher(
+ func(common.Hash) bool { return false },
+ nil,
+ func(string, []common.Hash) error { return nil },
+ )
+ },
+ steps: []interface{}{
+ // Announce all the transactions, wait a bit and ensure only a small
+ // percentage gets requested
+ doTxNotify{peer: "A", hashes: hashes},
+ doWait{time: txArriveTimeout, step: true},
+ isWaiting(nil),
+ isScheduled{
+ tracking: map[string][]common.Hash{
+ "A": hashes,
+ },
+ fetching: map[string][]common.Hash{
+ "A": hashes[1643 : 1643+maxTxRetrievals],
+ },
+ },
+ },
+ })
+}
+
+// Tests that then number of transactions a peer is allowed to announce and/or
+// request at the same time is hard capped.
+func TestTransactionFetcherDoSProtection(t *testing.T) {
+ // Create a slew of transactions and to announce them
+ var hashesA []common.Hash
+ for i := 0; i < maxTxAnnounces+1; i++ {
+ hashesA = append(hashesA, common.Hash{0x01, byte(i / 256), byte(i % 256)})
+ }
+ var hashesB []common.Hash
+ for i := 0; i < maxTxAnnounces+1; i++ {
+ hashesB = append(hashesB, common.Hash{0x02, byte(i / 256), byte(i % 256)})
+ }
+ testTransactionFetcherParallel(t, txFetcherTest{
+ init: func() *TxFetcher {
+ return NewTxFetcher(
+ func(common.Hash) bool { return false },
+ nil,
+ func(string, []common.Hash) error { return nil },
+ )
+ },
+ steps: []interface{}{
+ // Announce half of the transaction and wait for them to be scheduled
+ doTxNotify{peer: "A", hashes: hashesA[:maxTxAnnounces/2]},
+ doTxNotify{peer: "B", hashes: hashesB[:maxTxAnnounces/2-1]},
+ doWait{time: txArriveTimeout, step: true},
+
+ // Announce the second half and keep them in the wait list
+ doTxNotify{peer: "A", hashes: hashesA[maxTxAnnounces/2 : maxTxAnnounces]},
+ doTxNotify{peer: "B", hashes: hashesB[maxTxAnnounces/2-1 : maxTxAnnounces-1]},
+
+ // Ensure the hashes are split half and half
+ isWaiting(map[string][]common.Hash{
+ "A": hashesA[maxTxAnnounces/2 : maxTxAnnounces],
+ "B": hashesB[maxTxAnnounces/2-1 : maxTxAnnounces-1],
+ }),
+ isScheduled{
+ tracking: map[string][]common.Hash{
+ "A": hashesA[:maxTxAnnounces/2],
+ "B": hashesB[:maxTxAnnounces/2-1],
+ },
+ fetching: map[string][]common.Hash{
+ "A": hashesA[1643 : 1643+maxTxRetrievals],
+ "B": append(append([]common.Hash{}, hashesB[maxTxAnnounces/2-3:maxTxAnnounces/2-1]...), hashesB[:maxTxRetrievals-2]...),
+ },
+ },
+ // Ensure that adding even one more hash results in dropping the hash
+ doTxNotify{peer: "A", hashes: []common.Hash{hashesA[maxTxAnnounces]}},
+ doTxNotify{peer: "B", hashes: hashesB[maxTxAnnounces-1 : maxTxAnnounces+1]},
+
+ isWaiting(map[string][]common.Hash{
+ "A": hashesA[maxTxAnnounces/2 : maxTxAnnounces],
+ "B": hashesB[maxTxAnnounces/2-1 : maxTxAnnounces],
+ }),
+ isScheduled{
+ tracking: map[string][]common.Hash{
+ "A": hashesA[:maxTxAnnounces/2],
+ "B": hashesB[:maxTxAnnounces/2-1],
+ },
+ fetching: map[string][]common.Hash{
+ "A": hashesA[1643 : 1643+maxTxRetrievals],
+ "B": append(append([]common.Hash{}, hashesB[maxTxAnnounces/2-3:maxTxAnnounces/2-1]...), hashesB[:maxTxRetrievals-2]...),
+ },
+ },
+ },
+ })
+}
+
+// Tests that underpriced transactions don't get rescheduled after being rejected.
+func TestTransactionFetcherUnderpricedDedup(t *testing.T) {
+ testTransactionFetcherParallel(t, txFetcherTest{
+ init: func() *TxFetcher {
+ return NewTxFetcher(
+ func(common.Hash) bool { return false },
+ func(txs []*types.Transaction) []error {
+ errs := make([]error, len(txs))
+ for i := 0; i < len(errs); i++ {
+ if i%2 == 0 {
+ errs[i] = core.ErrUnderpriced
+ } else {
+ errs[i] = core.ErrReplaceUnderpriced
+ }
+ }
+ return errs
+ },
+ func(string, []common.Hash) error { return nil },
+ )
+ },
+ steps: []interface{}{
+ // Deliver a transaction through the fetcher, but reject as underpriced
+ doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0], testTxsHashes[1]}},
+ doWait{time: txArriveTimeout, step: true},
+ doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[0], testTxs[1]}, direct: true},
+ isScheduled{nil, nil, nil},
+
+ // Try to announce the transaction again, ensure it's not scheduled back
+ doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0], testTxsHashes[1], testTxsHashes[2]}}, // [2] is needed to force a step in the fetcher
+ isWaiting(map[string][]common.Hash{
+ "A": {testTxsHashes[2]},
+ }),
+ isScheduled{nil, nil, nil},
+ },
+ })
+}
+
+// Tests that underpriced transactions don't get rescheduled after being rejected,
+// but at the same time there's a hard cap on the number of transactions that are
+// tracked.
+func TestTransactionFetcherUnderpricedDoSProtection(t *testing.T) {
+ // Temporarily disable fetch timeouts as they massively mess up the simulated clock
+ defer func(timeout time.Duration) { txFetchTimeout = timeout }(txFetchTimeout)
+ txFetchTimeout = 24 * time.Hour
+
+ // Create a slew of transactions to max out the underpriced set
+ var txs []*types.Transaction
+ for i := 0; i < maxTxUnderpricedSetSize+1; i++ {
+ txs = append(txs, types.NewTransaction(rand.Uint64(), common.Address{byte(rand.Intn(256))}, new(big.Int), 0, new(big.Int), nil))
+ }
+ hashes := make([]common.Hash, len(txs))
+ for i, tx := range txs {
+ hashes[i] = tx.Hash()
+ }
+ // Generate a set of steps to announce and deliver the entire set of transactions
+ var steps []interface{}
+ for i := 0; i < maxTxUnderpricedSetSize/maxTxRetrievals; i++ {
+ steps = append(steps, doTxNotify{peer: "A", hashes: hashes[i*maxTxRetrievals : (i+1)*maxTxRetrievals]})
+ steps = append(steps, isWaiting(map[string][]common.Hash{
+ "A": hashes[i*maxTxRetrievals : (i+1)*maxTxRetrievals],
+ }))
+ steps = append(steps, doWait{time: txArriveTimeout, step: true})
+ steps = append(steps, isScheduled{
+ tracking: map[string][]common.Hash{
+ "A": hashes[i*maxTxRetrievals : (i+1)*maxTxRetrievals],
+ },
+ fetching: map[string][]common.Hash{
+ "A": hashes[i*maxTxRetrievals : (i+1)*maxTxRetrievals],
+ },
+ })
+ steps = append(steps, doTxEnqueue{peer: "A", txs: txs[i*maxTxRetrievals : (i+1)*maxTxRetrievals], direct: true})
+ steps = append(steps, isWaiting(nil))
+ steps = append(steps, isScheduled{nil, nil, nil})
+ steps = append(steps, isUnderpriced((i+1)*maxTxRetrievals))
+ }
+ testTransactionFetcher(t, txFetcherTest{
+ init: func() *TxFetcher {
+ return NewTxFetcher(
+ func(common.Hash) bool { return false },
+ func(txs []*types.Transaction) []error {
+ errs := make([]error, len(txs))
+ for i := 0; i < len(errs); i++ {
+ errs[i] = core.ErrUnderpriced
+ }
+ return errs
+ },
+ func(string, []common.Hash) error { return nil },
+ )
+ },
+ steps: append(steps, []interface{}{
+ // The preparation of the test has already been done in `steps`, add the last check
+ doTxNotify{peer: "A", hashes: []common.Hash{hashes[maxTxUnderpricedSetSize]}},
+ doWait{time: txArriveTimeout, step: true},
+ doTxEnqueue{peer: "A", txs: []*types.Transaction{txs[maxTxUnderpricedSetSize]}, direct: true},
+ isUnderpriced(maxTxUnderpricedSetSize),
+ }...),
+ })
+}
+
+// Tests that unexpected deliveries don't corrupt the internal state.
+func TestTransactionFetcherOutOfBoundDeliveries(t *testing.T) {
+ testTransactionFetcherParallel(t, txFetcherTest{
+ init: func() *TxFetcher {
+ return NewTxFetcher(
+ func(common.Hash) bool { return false },
+ func(txs []*types.Transaction) []error {
+ return make([]error, len(txs))
+ },
+ func(string, []common.Hash) error { return nil },
+ )
+ },
+ steps: []interface{}{
+ // Deliver something out of the blue
+ isWaiting(nil),
+ isScheduled{nil, nil, nil},
+ doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[0]}, direct: false},
+ isWaiting(nil),
+ isScheduled{nil, nil, nil},
+
+ // Set up a few hashes into various stages
+ doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}},
+ doWait{time: txArriveTimeout, step: true},
+ doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[1]}},
+ doWait{time: txArriveTimeout, step: true},
+ doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[2]}},
+
+ isWaiting(map[string][]common.Hash{
+ "A": {testTxsHashes[2]},
+ }),
+ isScheduled{
+ tracking: map[string][]common.Hash{
+ "A": {testTxsHashes[0], testTxsHashes[1]},
+ },
+ fetching: map[string][]common.Hash{
+ "A": {testTxsHashes[0]},
+ },
+ },
+ // Deliver everything and more out of the blue
+ doTxEnqueue{peer: "B", txs: []*types.Transaction{testTxs[0], testTxs[1], testTxs[2], testTxs[3]}, direct: true},
+ isWaiting(nil),
+ isScheduled{
+ tracking: nil,
+ fetching: nil,
+ dangling: map[string][]common.Hash{
+ "A": {testTxsHashes[0]},
+ },
+ },
+ },
+ })
+}
+
+// Tests that dropping a peer cleans out all internal data structures in all the
+// live or danglng stages.
+func TestTransactionFetcherDrop(t *testing.T) {
+ testTransactionFetcherParallel(t, txFetcherTest{
+ init: func() *TxFetcher {
+ return NewTxFetcher(
+ func(common.Hash) bool { return false },
+ func(txs []*types.Transaction) []error {
+ return make([]error, len(txs))
+ },
+ func(string, []common.Hash) error { return nil },
+ )
+ },
+ steps: []interface{}{
+ // Set up a few hashes into various stages
+ doTxNotify{peer: "A", hashes: []common.Hash{{0x01}}},
+ doWait{time: txArriveTimeout, step: true},
+ doTxNotify{peer: "A", hashes: []common.Hash{{0x02}}},
+ doWait{time: txArriveTimeout, step: true},
+ doTxNotify{peer: "A", hashes: []common.Hash{{0x03}}},
+
+ isWaiting(map[string][]common.Hash{
+ "A": {{0x03}},
+ }),
+ isScheduled{
+ tracking: map[string][]common.Hash{
+ "A": {{0x01}, {0x02}},
+ },
+ fetching: map[string][]common.Hash{
+ "A": {{0x01}},
+ },
+ },
+ // Drop the peer and ensure everything's cleaned out
+ doDrop("A"),
+ isWaiting(nil),
+ isScheduled{nil, nil, nil},
+
+ // Push the node into a dangling (timeout) state
+ doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}},
+ doWait{time: txArriveTimeout, step: true},
+ isWaiting(nil),
+ isScheduled{
+ tracking: map[string][]common.Hash{
+ "A": {testTxsHashes[0]},
+ },
+ fetching: map[string][]common.Hash{
+ "A": {testTxsHashes[0]},
+ },
+ },
+ doWait{time: txFetchTimeout, step: true},
+ isWaiting(nil),
+ isScheduled{
+ tracking: nil,
+ fetching: nil,
+ dangling: map[string][]common.Hash{
+ "A": {},
+ },
+ },
+ // Drop the peer and ensure everything's cleaned out
+ doDrop("A"),
+ isWaiting(nil),
+ isScheduled{nil, nil, nil},
+ },
+ })
+}
+
+// Tests that dropping a peer instantly reschedules failed announcements to any
+// available peer.
+func TestTransactionFetcherDropRescheduling(t *testing.T) {
+ testTransactionFetcherParallel(t, txFetcherTest{
+ init: func() *TxFetcher {
+ return NewTxFetcher(
+ func(common.Hash) bool { return false },
+ func(txs []*types.Transaction) []error {
+ return make([]error, len(txs))
+ },
+ func(string, []common.Hash) error { return nil },
+ )
+ },
+ steps: []interface{}{
+ // Set up a few hashes into various stages
+ doTxNotify{peer: "A", hashes: []common.Hash{{0x01}}},
+ doWait{time: txArriveTimeout, step: true},
+ doTxNotify{peer: "B", hashes: []common.Hash{{0x01}}},
+
+ isWaiting(nil),
+ isScheduled{
+ tracking: map[string][]common.Hash{
+ "A": {{0x01}},
+ "B": {{0x01}},
+ },
+ fetching: map[string][]common.Hash{
+ "A": {{0x01}},
+ },
+ },
+ // Drop the peer and ensure everything's cleaned out
+ doDrop("A"),
+ isWaiting(nil),
+ isScheduled{
+ tracking: map[string][]common.Hash{
+ "B": {{0x01}},
+ },
+ fetching: map[string][]common.Hash{
+ "B": {{0x01}},
+ },
+ },
+ },
+ })
+}
+
+// This test reproduces a crash caught by the fuzzer. The root cause was a
+// dangling transaction timing out and clashing on readd with a concurrently
+// announced one.
+func TestTransactionFetcherFuzzCrash01(t *testing.T) {
+ testTransactionFetcherParallel(t, txFetcherTest{
+ init: func() *TxFetcher {
+ return NewTxFetcher(
+ func(common.Hash) bool { return false },
+ func(txs []*types.Transaction) []error {
+ return make([]error, len(txs))
+ },
+ func(string, []common.Hash) error { return nil },
+ )
+ },
+ steps: []interface{}{
+ // Get a transaction into fetching mode and make it dangling with a broadcast
+ doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}},
+ doWait{time: txArriveTimeout, step: true},
+ doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[0]}},
+
+ // Notify the dangling transaction once more and crash via a timeout
+ doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}},
+ doWait{time: txFetchTimeout, step: true},
+ },
+ })
+}
+
+// This test reproduces a crash caught by the fuzzer. The root cause was a
+// dangling transaction getting peer-dropped and clashing on readd with a
+// concurrently announced one.
+func TestTransactionFetcherFuzzCrash02(t *testing.T) {
+ testTransactionFetcherParallel(t, txFetcherTest{
+ init: func() *TxFetcher {
+ return NewTxFetcher(
+ func(common.Hash) bool { return false },
+ func(txs []*types.Transaction) []error {
+ return make([]error, len(txs))
+ },
+ func(string, []common.Hash) error { return nil },
+ )
+ },
+ steps: []interface{}{
+ // Get a transaction into fetching mode and make it dangling with a broadcast
+ doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}},
+ doWait{time: txArriveTimeout, step: true},
+ doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[0]}},
+
+ // Notify the dangling transaction once more, re-fetch, and crash via a drop and timeout
+ doTxNotify{peer: "B", hashes: []common.Hash{testTxsHashes[0]}},
+ doWait{time: txArriveTimeout, step: true},
+ doDrop("A"),
+ doWait{time: txFetchTimeout, step: true},
+ },
+ })
+}
+
+// This test reproduces a crash caught by the fuzzer. The root cause was a
+// dangling transaction getting rescheduled via a partial delivery, clashing
+// with a concurrent notify.
+func TestTransactionFetcherFuzzCrash03(t *testing.T) {
+ testTransactionFetcherParallel(t, txFetcherTest{
+ init: func() *TxFetcher {
+ return NewTxFetcher(
+ func(common.Hash) bool { return false },
+ func(txs []*types.Transaction) []error {
+ return make([]error, len(txs))
+ },
+ func(string, []common.Hash) error { return nil },
+ )
+ },
+ steps: []interface{}{
+ // Get a transaction into fetching mode and make it dangling with a broadcast
+ doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0], testTxsHashes[1]}},
+ doWait{time: txFetchTimeout, step: true},
+ doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[0], testTxs[1]}},
+
+ // Notify the dangling transaction once more, partially deliver, clash&crash with a timeout
+ doTxNotify{peer: "B", hashes: []common.Hash{testTxsHashes[0]}},
+ doWait{time: txArriveTimeout, step: true},
+
+ doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[1]}, direct: true},
+ doWait{time: txFetchTimeout, step: true},
+ },
+ })
+}
+
+// This test reproduces a crash caught by the fuzzer. The root cause was a
+// dangling transaction getting rescheduled via a disconnect, clashing with
+// a concurrent notify.
+func TestTransactionFetcherFuzzCrash04(t *testing.T) {
+ // Create a channel to control when tx requests can fail
+ proceed := make(chan struct{})
+
+ testTransactionFetcherParallel(t, txFetcherTest{
+ init: func() *TxFetcher {
+ return NewTxFetcher(
+ func(common.Hash) bool { return false },
+ func(txs []*types.Transaction) []error {
+ return make([]error, len(txs))
+ },
+ func(string, []common.Hash) error {
+ <-proceed
+ return errors.New("peer disconnected")
+ },
+ )
+ },
+ steps: []interface{}{
+ // Get a transaction into fetching mode and make it dangling with a broadcast
+ doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}},
+ doWait{time: txArriveTimeout, step: true},
+ doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[0]}},
+
+ // Notify the dangling transaction once more, re-fetch, and crash via an in-flight disconnect
+ doTxNotify{peer: "B", hashes: []common.Hash{testTxsHashes[0]}},
+ doWait{time: txArriveTimeout, step: true},
+ doFunc(func() {
+ proceed <- struct{}{} // Allow peer A to return the failure
+ }),
+ doWait{time: 0, step: true},
+ doWait{time: txFetchTimeout, step: true},
+ },
+ })
+}
+
+func testTransactionFetcherParallel(t *testing.T, tt txFetcherTest) {
+ t.Parallel()
+ testTransactionFetcher(t, tt)
+}
+
+func testTransactionFetcher(t *testing.T, tt txFetcherTest) {
+ // Create a fetcher and hook into it's simulated fields
+ clock := new(mclock.Simulated)
+ wait := make(chan struct{})
+
+ fetcher := tt.init()
+ fetcher.clock = clock
+ fetcher.step = wait
+ fetcher.rand = rand.New(rand.NewSource(0x3a29))
+
+ fetcher.Start()
+ defer fetcher.Stop()
+
+ // Crunch through all the test steps and execute them
+ for i, step := range tt.steps {
+ switch step := step.(type) {
+ case doTxNotify:
+ if err := fetcher.Notify(step.peer, step.hashes); err != nil {
+ t.Errorf("step %d: %v", i, err)
+ }
+ <-wait // Fetcher needs to process this, wait until it's done
+ select {
+ case <-wait:
+ panic("wtf")
+ case <-time.After(time.Millisecond):
+ }
+
+ case doTxEnqueue:
+ if err := fetcher.Enqueue(step.peer, step.txs, step.direct); err != nil {
+ t.Errorf("step %d: %v", i, err)
+ }
+ <-wait // Fetcher needs to process this, wait until it's done
+
+ case doWait:
+ clock.Run(step.time)
+ if step.step {
+ <-wait // Fetcher supposed to do something, wait until it's done
+ }
+
+ case doDrop:
+ if err := fetcher.Drop(string(step)); err != nil {
+ t.Errorf("step %d: %v", i, err)
+ }
+ <-wait // Fetcher needs to process this, wait until it's done
+
+ case doFunc:
+ step()
+
+ case isWaiting:
+ // We need to check that the waiting list (stage 1) internals
+ // match with the expected set. Check the peer->hash mappings
+ // first.
+ for peer, hashes := range step {
+ waiting := fetcher.waitslots[peer]
+ if waiting == nil {
+ t.Errorf("step %d: peer %s missing from waitslots", i, peer)
+ continue
+ }
+ for _, hash := range hashes {
+ if _, ok := waiting[hash]; !ok {
+ t.Errorf("step %d, peer %s: hash %x missing from waitslots", i, peer, hash)
+ }
+ }
+ for hash := range waiting {
+ if !containsHash(hashes, hash) {
+ t.Errorf("step %d, peer %s: hash %x extra in waitslots", i, peer, hash)
+ }
+ }
+ }
+ for peer := range fetcher.waitslots {
+ if _, ok := step[peer]; !ok {
+ t.Errorf("step %d: peer %s extra in waitslots", i, peer)
+ }
+ }
+ // Peer->hash sets correct, check the hash->peer and timeout sets
+ for peer, hashes := range step {
+ for _, hash := range hashes {
+ if _, ok := fetcher.waitlist[hash][peer]; !ok {
+ t.Errorf("step %d, hash %x: peer %s missing from waitlist", i, hash, peer)
+ }
+ if _, ok := fetcher.waittime[hash]; !ok {
+ t.Errorf("step %d: hash %x missing from waittime", i, hash)
+ }
+ }
+ }
+ for hash, peers := range fetcher.waitlist {
+ if len(peers) == 0 {
+ t.Errorf("step %d, hash %x: empty peerset in waitlist", i, hash)
+ }
+ for peer := range peers {
+ if !containsHash(step[peer], hash) {
+ t.Errorf("step %d, hash %x: peer %s extra in waitlist", i, hash, peer)
+ }
+ }
+ }
+ for hash := range fetcher.waittime {
+ var found bool
+ for _, hashes := range step {
+ if containsHash(hashes, hash) {
+ found = true
+ break
+ }
+ }
+ if !found {
+ t.Errorf("step %d,: hash %x extra in waittime", i, hash)
+ }
+ }
+
+ case isScheduled:
+ // Check that all scheduled announces are accounted for and no
+ // extra ones are present.
+ for peer, hashes := range step.tracking {
+ scheduled := fetcher.announces[peer]
+ if scheduled == nil {
+ t.Errorf("step %d: peer %s missing from announces", i, peer)
+ continue
+ }
+ for _, hash := range hashes {
+ if _, ok := scheduled[hash]; !ok {
+ t.Errorf("step %d, peer %s: hash %x missing from announces", i, peer, hash)
+ }
+ }
+ for hash := range scheduled {
+ if !containsHash(hashes, hash) {
+ t.Errorf("step %d, peer %s: hash %x extra in announces", i, peer, hash)
+ }
+ }
+ }
+ for peer := range fetcher.announces {
+ if _, ok := step.tracking[peer]; !ok {
+ t.Errorf("step %d: peer %s extra in announces", i, peer)
+ }
+ }
+ // Check that all announces required to be fetching are in the
+ // appropriate sets
+ for peer, hashes := range step.fetching {
+ request := fetcher.requests[peer]
+ if request == nil {
+ t.Errorf("step %d: peer %s missing from requests", i, peer)
+ continue
+ }
+ for _, hash := range hashes {
+ if !containsHash(request.hashes, hash) {
+ t.Errorf("step %d, peer %s: hash %x missing from requests", i, peer, hash)
+ }
+ }
+ for _, hash := range request.hashes {
+ if !containsHash(hashes, hash) {
+ t.Errorf("step %d, peer %s: hash %x extra in requests", i, peer, hash)
+ }
+ }
+ }
+ for peer := range fetcher.requests {
+ if _, ok := step.fetching[peer]; !ok {
+ if _, ok := step.dangling[peer]; !ok {
+ t.Errorf("step %d: peer %s extra in requests", i, peer)
+ }
+ }
+ }
+ for peer, hashes := range step.fetching {
+ for _, hash := range hashes {
+ if _, ok := fetcher.fetching[hash]; !ok {
+ t.Errorf("step %d, peer %s: hash %x missing from fetching", i, peer, hash)
+ }
+ }
+ }
+ for hash := range fetcher.fetching {
+ var found bool
+ for _, req := range fetcher.requests {
+ if containsHash(req.hashes, hash) {
+ found = true
+ break
+ }
+ }
+ if !found {
+ t.Errorf("step %d: hash %x extra in fetching", i, hash)
+ }
+ }
+ for _, hashes := range step.fetching {
+ for _, hash := range hashes {
+ alternates := fetcher.alternates[hash]
+ if alternates == nil {
+ t.Errorf("step %d: hash %x missing from alternates", i, hash)
+ continue
+ }
+ for peer := range alternates {
+ if _, ok := fetcher.announces[peer]; !ok {
+ t.Errorf("step %d: peer %s extra in alternates", i, peer)
+ continue
+ }
+ if _, ok := fetcher.announces[peer][hash]; !ok {
+ t.Errorf("step %d, peer %s: hash %x extra in alternates", i, hash, peer)
+ continue
+ }
+ }
+ for p := range fetcher.announced[hash] {
+ if _, ok := alternates[p]; !ok {
+ t.Errorf("step %d, hash %x: peer %s missing from alternates", i, hash, p)
+ continue
+ }
+ }
+ }
+ }
+ for peer, hashes := range step.dangling {
+ request := fetcher.requests[peer]
+ if request == nil {
+ t.Errorf("step %d: peer %s missing from requests", i, peer)
+ continue
+ }
+ for _, hash := range hashes {
+ if !containsHash(request.hashes, hash) {
+ t.Errorf("step %d, peer %s: hash %x missing from requests", i, peer, hash)
+ }
+ }
+ for _, hash := range request.hashes {
+ if !containsHash(hashes, hash) {
+ t.Errorf("step %d, peer %s: hash %x extra in requests", i, peer, hash)
+ }
+ }
+ }
+ // Check that all transaction announces that are scheduled for
+ // retrieval but not actively being downloaded are tracked only
+ // in the stage 2 `announced` map.
+ var queued []common.Hash
+ for _, hashes := range step.tracking {
+ for _, hash := range hashes {
+ var found bool
+ for _, hs := range step.fetching {
+ if containsHash(hs, hash) {
+ found = true
+ break
+ }
+ }
+ if !found {
+ queued = append(queued, hash)
+ }
+ }
+ }
+ for _, hash := range queued {
+ if _, ok := fetcher.announced[hash]; !ok {
+ t.Errorf("step %d: hash %x missing from announced", i, hash)
+ }
+ }
+ for hash := range fetcher.announced {
+ if !containsHash(queued, hash) {
+ t.Errorf("step %d: hash %x extra in announced", i, hash)
+ }
+ }
+
+ case isUnderpriced:
+ if fetcher.underpriced.Cardinality() != int(step) {
+ t.Errorf("step %d: underpriced set size mismatch: have %d, want %d", i, fetcher.underpriced.Cardinality(), step)
+ }
+
+ default:
+ t.Fatalf("step %d: unknown step type %T", i, step)
+ }
+ // After every step, cross validate the internal uniqueness invariants
+ // between stage one and stage two.
+ for hash := range fetcher.waittime {
+ if _, ok := fetcher.announced[hash]; ok {
+ t.Errorf("step %d: hash %s present in both stage 1 and 2", i, hash)
+ }
+ }
+ }
+}
+
+// containsHash returns whether a hash is contained within a hash slice.
+func containsHash(slice []common.Hash, hash common.Hash) bool {
+ for _, have := range slice {
+ if have == hash {
+ return true
+ }
+ }
+ return false
+}
diff --git a/eth/handler.go b/eth/handler.go
index c6f991ca9469..786a28fb7e26 100644
--- a/eth/handler.go
+++ b/eth/handler.go
@@ -18,7 +18,9 @@ package eth
import (
"encoding/json"
+ "errors"
"fmt"
+ "math"
"math/big"
"sync"
"sync/atomic"
@@ -51,6 +53,9 @@ const (
// txChanSize is the size of channel listening to NewTxsEvent.
// The number is referenced from the size of tx pool.
txChanSize = 4096
+
+ // minimim number of peers to broadcast entire blocks and transactions too.
+ minBroadcastPeers = 4
)
var (
@@ -79,10 +84,11 @@ type ProtocolManager struct {
chainconfig *params.ChainConfig
maxPeers int
- downloader *downloader.Downloader
- fetcher *fetcher.Fetcher
- peers *peerSet
- bft *bft.Bfter
+ downloader *downloader.Downloader
+ blockFetcher *fetcher.BlockFetcher
+ txFetcher *fetcher.TxFetcher
+ peers *peerSet
+ bft *bft.Bfter
eventMux *event.TypeMux
txsCh chan core.NewTxsEvent
@@ -110,6 +116,9 @@ type ProtocolManager struct {
knownVotes *lru.Cache
knownSyncInfos *lru.Cache
knownTimeouts *lru.Cache
+
+ // Test fields or hooks
+ broadcastTxAnnouncesOnly bool // Testing field, disable transaction propagation
}
// NewProtocolManagerEx add order pool to protocol
@@ -237,14 +246,31 @@ func NewProtocolManager(config *params.ChainConfig, mode downloader.SyncMode, ne
return blockchain.CurrentBlock().NumberU64()
}
- inserter := func(block *types.Block) error {
- // If fast sync is running, deny importing weird blocks
+ inserter := func(blocks types.Blocks) (int, error) {
+ // If sync hasn't reached the checkpoint yet, deny importing weird blocks.
+ //
+ // Ideally we would also compare the head block's timestamp and similarly reject
+ // the propagated block if the head is too old. Unfortunately there is a corner
+ // case when starting new networks, where the genesis might be ancient (0 unix)
+ // which would prevent full nodes from accepting it.
+ if manager.blockchain.CurrentBlock().NumberU64() < manager.checkpointNumber {
+ log.Warn("Unsynced yet, discarded propagated block", "number", blocks[0].Number(), "hash", blocks[0].Hash())
+ return 0, nil
+ }
+ // If fast sync is running, deny importing weird blocks. This is a problematic
+ // clause when starting up a new network, because fast-syncing miners might not
+ // accept each others' blocks until a restart. Unfortunately we haven't figured
+ // out a way yet where nodes can decide unilaterally whether the network is new
+ // or not. This should be fixed if we figure out a solution.
if atomic.LoadUint32(&manager.fastSync) == 1 {
- log.Warn("Discarded bad propagated block", "number", block.Number(), "hash", block.Hash())
- return nil
+ log.Warn("Fast syncing, discarded propagated block", "number", blocks[0].Number(), "hash", blocks[0].Hash())
+ return 0, nil
}
- atomic.StoreUint32(&manager.acceptTxs, 1) // Mark initial sync done on any fetcher import
- return manager.blockchain.InsertBlock(block)
+ n, err := manager.blockchain.InsertChain(blocks)
+ if err == nil {
+ atomic.StoreUint32(&manager.acceptTxs, 1) // Mark initial sync done on any fetcher import
+ }
+ return n, err
}
prepare := func(block *types.Block) error {
@@ -256,7 +282,7 @@ func NewProtocolManager(config *params.ChainConfig, mode downloader.SyncMode, ne
atomic.StoreUint32(&manager.acceptTxs, 1) // Mark initial sync done on any fetcher import
return manager.blockchain.PrepareBlock(block)
}
- manager.fetcher = fetcher.New(blockchain.GetBlockByHash, validator, handleProposedBlock, manager.BroadcastBlock, heighter, inserter, prepare, manager.removePeer)
+ manager.blockFetcher = fetcher.NewBlockFetcher(blockchain.GetBlockByHash, validator, handleProposedBlock, manager.BroadcastBlock, heighter, inserter, prepare, manager.removePeer)
//Define bft function
broadcasts := bft.BroadcastFns{
Vote: manager.BroadcastVote,
@@ -269,6 +295,15 @@ func NewProtocolManager(config *params.ChainConfig, mode downloader.SyncMode, ne
manager.bft.SetConsensusFuns(engine)
}
+ fetchTx := func(peer string, hashes []common.Hash) error {
+ p := manager.peers.Peer(peer)
+ if p == nil {
+ return errors.New("unknown peer")
+ }
+ return p.RequestTxs(hashes)
+ }
+ manager.txFetcher = fetcher.NewTxFetcher(txpool.Has, txpool.AddRemotes, fetchTx)
+
return manager, nil
}
@@ -290,7 +325,7 @@ func (pm *ProtocolManager) makeProtocol(version uint) p2p.Protocol {
Version: version,
Length: length,
Run: func(p *p2p.Peer, rw p2p.MsgReadWriter) error {
- peer := pm.newPeer(int(version), p, rw)
+ peer := pm.newPeer(int(version), p, rw, pm.txpool.Get)
select {
case pm.newPeerCh <- peer:
pm.wg.Add(1)
@@ -322,6 +357,8 @@ func (pm *ProtocolManager) removePeer(id string) {
// Unregister the peer from the downloader and Ethereum peer set
pm.downloader.UnregisterPeer(id)
+ pm.txFetcher.Drop(id)
+
if err := pm.peers.Unregister(id); err != nil {
log.Debug("Peer removal failed", "peer", id, "err", err)
}
@@ -354,7 +391,7 @@ func (pm *ProtocolManager) Start(maxPeers int) {
// start sync handlers
go pm.syncer()
- go pm.txsyncLoop()
+ go pm.txsyncLoop64() // TODO(karalabe): Legacy initial tx echange, drop with eth/64.
}
func (pm *ProtocolManager) Stop() {
@@ -388,8 +425,8 @@ func (pm *ProtocolManager) Stop() {
log.Info("Ethereum protocol stopped")
}
-func (pm *ProtocolManager) newPeer(pv int, p *p2p.Peer, rw p2p.MsgReadWriter) *peer {
- return newPeer(pv, p, newMeteredMsgWriter(rw))
+func (pm *ProtocolManager) newPeer(pv int, p *p2p.Peer, rw p2p.MsgReadWriter, getPooledTx func(hash common.Hash) *types.Transaction) *peer {
+ return newPeer(pv, p, rw, getPooledTx)
}
// handle is the callback invoked to manage the life cycle of an eth peer. When
@@ -413,9 +450,6 @@ func (pm *ProtocolManager) handle(p *peer) error {
p.Log().Debug("Ethereum handshake failed", "err", err)
return err
}
- if rw, ok := p.rw.(*meteredMsgReadWriter); ok {
- rw.Init(p.version)
- }
// Register the peer locally
err := pm.peers.Register(p)
if err != nil && err != p2p.ErrAddPairPeer {
@@ -603,7 +637,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
return nil
}
// Irrelevant of the fork checks, send the header to the fetcher just in case
- headers = pm.fetcher.FilterHeaders(p.id, headers, time.Now())
+ headers = pm.blockFetcher.FilterHeaders(p.id, headers, time.Now())
}
if len(headers) > 0 || !filter {
err := pm.downloader.DeliverHeaders(p.id, headers)
@@ -646,20 +680,20 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
return errResp(ErrDecode, "msg %v: %v", msg, err)
}
// Deliver them all to the downloader for queuing
- trasactions := make([][]*types.Transaction, len(request))
+ transactions := make([][]*types.Transaction, len(request))
uncles := make([][]*types.Header, len(request))
for i, body := range request {
- trasactions[i] = body.Transactions
+ transactions[i] = body.Transactions
uncles[i] = body.Uncles
}
// Filter out any explicitly requested bodies, deliver the rest to the downloader
- filter := len(trasactions) > 0 || len(uncles) > 0
+ filter := len(transactions) > 0 || len(uncles) > 0
if filter {
- trasactions, uncles = pm.fetcher.FilterBodies(p.id, trasactions, uncles, time.Now())
+ transactions, uncles = pm.blockFetcher.FilterBodies(p.id, transactions, uncles, time.Now())
}
- if len(trasactions) > 0 || len(uncles) > 0 || !filter {
- err := pm.downloader.DeliverBodies(p.id, trasactions, uncles)
+ if len(transactions) > 0 || len(uncles) > 0 || !filter {
+ err := pm.downloader.DeliverBodies(p.id, transactions, uncles)
if err != nil {
log.Debug("Failed to deliver bodies", "err", err)
}
@@ -767,7 +801,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
}
}
for _, block := range unknown {
- pm.fetcher.Notify(p.id, block.Hash, block.Number, time.Now(), p.RequestOneHeader, p.RequestBodies)
+ pm.blockFetcher.Notify(p.id, block.Hash, block.Number, time.Now(), p.RequestOneHeader, p.RequestBodies)
}
case msg.Code == NewBlockMsg:
@@ -781,7 +815,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
// Mark the peer as owning the block and schedule it for import
p.MarkBlock(request.Block.Hash())
- pm.fetcher.Enqueue(p.id, request.Block)
+ pm.blockFetcher.Enqueue(p.id, request.Block)
// Assuming the block is importable by the peer, but possibly not yet done so,
// calculate the head hash and TD that the peer truly must have.
@@ -802,7 +836,59 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
}
}
- case msg.Code == TxMsg:
+ case msg.Code == NewPooledTransactionHashesMsg && p.version >= eth65:
+ // New transaction announcement arrived, make sure we have
+ // a valid and fresh chain to handle them
+ if atomic.LoadUint32(&pm.acceptTxs) == 0 {
+ break
+ }
+ var hashes []common.Hash
+ if err := msg.Decode(&hashes); err != nil {
+ return errResp(ErrDecode, "msg %v: %v", msg, err)
+ }
+ // Schedule all the unknown hashes for retrieval
+ for _, hash := range hashes {
+ p.MarkTransaction(hash)
+ }
+ pm.txFetcher.Notify(p.id, hashes)
+
+ case msg.Code == GetPooledTransactionsMsg && p.version >= eth65:
+ // Decode the retrieval message
+ msgStream := rlp.NewStream(msg.Payload, uint64(msg.Size))
+ if _, err := msgStream.List(); err != nil {
+ return err
+ }
+ // Gather transactions until the fetch or network limits is reached
+ var (
+ hash common.Hash
+ bytes int
+ hashes []common.Hash
+ txs []rlp.RawValue
+ )
+ for bytes < softResponseLimit {
+ // Retrieve the hash of the next block
+ if err := msgStream.Decode(&hash); err == rlp.EOL {
+ break
+ } else if err != nil {
+ return errResp(ErrDecode, "msg %v: %v", msg, err)
+ }
+ // Retrieve the requested transaction, skipping if unknown to us
+ tx := pm.txpool.Get(hash)
+ if tx == nil {
+ continue
+ }
+ // If known, encode and queue for response packet
+ if encoded, err := rlp.EncodeToBytes(tx); err != nil {
+ log.Error("Failed to encode transaction", "err", err)
+ } else {
+ hashes = append(hashes, hash)
+ txs = append(txs, encoded)
+ bytes += len(encoded)
+ }
+ }
+ return p.SendPooledTransactionsRLP(hashes, txs)
+
+ case msg.Code == TransactionMsg || (msg.Code == PooledTransactionsMsg && p.version >= eth65):
// Transactions arrived, make sure we have a valid and fresh chain to handle them
if atomic.LoadUint32(&pm.acceptTxs) == 0 {
break
@@ -828,7 +914,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
}
}
- pm.txpool.AddRemotes(txs)
+ pm.txFetcher.Enqueue(p.id, txs, msg.Code == PooledTransactionsMsg)
case msg.Code == OrderTxMsg:
// Transactions arrived, make sure we have a valid and fresh chain to handle them
@@ -984,22 +1070,50 @@ func (pm *ProtocolManager) BroadcastBlock(block *types.Block, propagate bool) {
}
}
-// BroadcastTxs will propagate a batch of transactions to all peers which are not known to
+// BroadcastTransactions will propagate a batch of transactions to all peers which are not known to
// already have the given transaction.
-func (pm *ProtocolManager) BroadcastTxs(txs types.Transactions) {
- var txset = make(map[*peer]types.Transactions)
-
+func (pm *ProtocolManager) BroadcastTransactions(txs types.Transactions, propagate bool) {
+ var (
+ txset = make(map[*peer][]common.Hash)
+ annos = make(map[*peer][]common.Hash)
+ )
// Broadcast transactions to a batch of peers not knowing about it
+ if propagate {
+ for _, tx := range txs {
+ peers := pm.peers.PeersWithoutTx(tx.Hash())
+
+ // Send the block to a subset of our peers
+ transferLen := int(math.Sqrt(float64(len(peers))))
+ if transferLen < minBroadcastPeers {
+ transferLen = minBroadcastPeers
+ }
+ if transferLen > len(peers) {
+ transferLen = len(peers)
+ }
+ transfer := peers[:transferLen]
+ for _, peer := range transfer {
+ txset[peer] = append(txset[peer], tx.Hash())
+ }
+ log.Trace("Broadcast transaction", "hash", tx.Hash(), "recipients", len(peers))
+ }
+ for peer, hashes := range txset {
+ peer.AsyncSendTransactions(hashes)
+ }
+ return
+ }
+ // Otherwise only broadcast the announcement to peers
for _, tx := range txs {
peers := pm.peers.PeersWithoutTx(tx.Hash())
for _, peer := range peers {
- txset[peer] = append(txset[peer], tx)
+ annos[peer] = append(annos[peer], tx.Hash())
}
- log.Trace("Broadcast transaction", "hash", tx.Hash(), "recipients", len(peers))
}
- // FIXME include this again: peers = peers[:int(math.Sqrt(float64(len(peers))))]
- for peer, txs := range txset {
- peer.SendTransactions(txs)
+ for peer, hashes := range annos {
+ if peer.version >= eth65 {
+ peer.AsyncSendPooledTransactionHashes(hashes)
+ } else {
+ peer.AsyncSendTransactions(hashes)
+ }
}
}
@@ -1095,7 +1209,13 @@ func (pm *ProtocolManager) txBroadcastLoop() {
for {
select {
case event := <-pm.txsCh:
- pm.BroadcastTxs(event.Txs)
+ // For testing purpose only, disable propagation
+ if pm.broadcastTxAnnouncesOnly {
+ pm.BroadcastTransactions(event.Txs, false)
+ continue
+ }
+ pm.BroadcastTransactions(event.Txs, true) // First propagate transactions to peers
+ pm.BroadcastTransactions(event.Txs, false) // Only then announce to the rest
// Err() channel will be closed when unsubscribing.
case <-pm.txsSub.Err():
diff --git a/eth/handler_test.go b/eth/handler_test.go
index 5a4c35d05fb3..ed3409c5a4b9 100644
--- a/eth/handler_test.go
+++ b/eth/handler_test.go
@@ -17,6 +17,7 @@
package eth
import (
+ "fmt"
"math"
"math/big"
"math/rand"
@@ -424,75 +425,245 @@ func testGetReceipt(t *testing.T, protocol int) {
}
}
-// Tests that post eth protocol handshake, DAO fork-enabled clients also execute
-// a DAO "challenge" verifying each others' DAO fork headers to ensure they're on
-// compatible chains.
-func TestDAOChallengeNoVsNo(t *testing.T) { testDAOChallenge(t, false, false, false) }
-func TestDAOChallengeNoVsPro(t *testing.T) { testDAOChallenge(t, false, true, false) }
-func TestDAOChallengeProVsNo(t *testing.T) { testDAOChallenge(t, true, false, false) }
-func TestDAOChallengeProVsPro(t *testing.T) { testDAOChallenge(t, true, true, false) }
-func TestDAOChallengeNoVsTimeout(t *testing.T) { testDAOChallenge(t, false, false, true) }
-func TestDAOChallengeProVsTimeout(t *testing.T) { testDAOChallenge(t, true, true, true) }
-
-func testDAOChallenge(t *testing.T, localForked, remoteForked bool, timeout bool) {
- // Reduce the DAO handshake challenge timeout
- if timeout {
- defer func(old time.Duration) { daoChallengeTimeout = old }(daoChallengeTimeout)
- daoChallengeTimeout = 500 * time.Millisecond
+// // Tests that post eth protocol handshake, DAO fork-enabled clients also execute
+// // a DAO "challenge" verifying each others' DAO fork headers to ensure they're on
+// // compatible chains.
+// func TestDAOChallengeNoVsNo(t *testing.T) { testDAOChallenge(t, false, false, false) }
+// func TestDAOChallengeNoVsPro(t *testing.T) { testDAOChallenge(t, false, true, false) }
+// func TestDAOChallengeProVsNo(t *testing.T) { testDAOChallenge(t, true, false, false) }
+// func TestDAOChallengeProVsPro(t *testing.T) { testDAOChallenge(t, true, true, false) }
+// func TestDAOChallengeNoVsTimeout(t *testing.T) { testDAOChallenge(t, false, false, true) }
+// func TestDAOChallengeProVsTimeout(t *testing.T) { testDAOChallenge(t, true, true, true) }
+
+// func testDAOChallenge(t *testing.T, localForked, remoteForked bool, timeout bool) {
+// // Reduce the DAO handshake challenge timeout
+// if timeout {
+// defer func(old time.Duration) { daoChallengeTimeout = old }(daoChallengeTimeout)
+// daoChallengeTimeout = 500 * time.Millisecond
+// }
+// // Create a DAO aware protocol manager
+// var (
+// evmux = new(event.TypeMux)
+// pow = ethash.NewFaker()
+// db = rawdb.NewMemoryDatabase()
+// config = ¶ms.ChainConfig{DAOForkBlock: big.NewInt(1), DAOForkSupport: localForked}
+// gspec = &core.Genesis{Config: config}
+// genesis = gspec.MustCommit(db)
+// blockchain, _ = core.NewBlockChain(db, nil, config, pow, vm.Config{})
+// )
+// (&core.Genesis{Config: config}).MustCommit(db) // Commit genesis block
+// // If checkpointing is enabled, create and inject a fake CHT and the corresponding
+// // chllenge response.
+// var response *types.Header
+// var cht *params.TrustedCheckpoint
+// if checkpoint {
+// index := uint64(rand.Intn(500))
+// number := (index+1)*params.CHTFrequency - 1
+// response = &types.Header{Number: big.NewInt(int64(number)), Extra: []byte("valid")}
+
+// cht = ¶ms.TrustedCheckpoint{
+// SectionIndex: index,
+// SectionHead: response.Hash(),
+// }
+// }
+// // Create a checkpoint aware protocol manager
+// blockchain, err := core.NewBlockChain(db, nil, config, ethash.NewFaker(), vm.Config{}, nil)
+// if err != nil {
+// t.Fatalf("failed to create new blockchain: %v", err)
+// }
+// // pm, err := NewProtocolManager(config, downloader.FullSync, DefaultConfig.NetworkId, evmux, new(testTxPool), pow, blockchain, db)
+// pm, err := NewProtocolManager(config, cht, syncmode, DefaultConfig.NetworkId, new(event.TypeMux), &testTxPool{pool: make(map[common.Hash]*types.Transaction)}, ethash.NewFaker(), blockchain, db, 1, nil)
+// if err != nil {
+// t.Fatalf("failed to start test protocol manager: %v", err)
+// }
+// pm.Start(1000)
+// defer pm.Stop()
+
+// // Connect a new peer and check that we receive the DAO challenge
+// peer, _ := newTestPeer("peer", eth63, pm, true)
+// defer peer.close()
+
+// challenge := &getBlockHeadersData{
+// Origin: hashOrNumber{Number: config.DAOForkBlock.Uint64()},
+// Amount: 1,
+// Skip: 0,
+// Reverse: false,
+// }
+// if err := p2p.ExpectMsg(peer.app, GetBlockHeadersMsg, challenge); err != nil {
+// t.Fatalf("challenge mismatch: %v", err)
+// }
+// // Create a block to reply to the challenge if no timeout is simulated
+// if !timeout {
+// blocks, _ := core.GenerateChain(¶ms.ChainConfig{}, genesis, ethash.NewFaker(), db, 1, func(i int, block *core.BlockGen) {
+// if remoteForked {
+// block.SetExtra(params.DAOForkBlockExtra)
+// }
+// })
+// if err := p2p.Send(peer.app, BlockHeadersMsg, []*types.Header{blocks[0].Header()}); err != nil {
+// t.Fatalf("failed to answer challenge: %v", err)
+// }
+// time.Sleep(100 * time.Millisecond) // Sleep to avoid the verification racing with the drops
+// } else {
+// // Otherwise wait until the test timeout passes
+// time.Sleep(daoChallengeTimeout + 500*time.Millisecond)
+// }
+// // Verify that depending on fork side, the remote peer is maintained or dropped
+// if localForked == remoteForked && !timeout {
+// if peers := pm.peers.Len(); peers != 1 {
+// t.Fatalf("peer count mismatch: have %d, want %d", peers, 1)
+// }
+// } else {
+// if peers := pm.peers.Len(); peers != 0 {
+// t.Fatalf("peer count mismatch: have %d, want %d", peers, 0)
+// }
+// }
+// }
+
+func TestBroadcastBlock(t *testing.T) {
+ var tests = []struct {
+ totalPeers int
+ broadcastExpected int
+ }{
+ {1, 1},
+ {2, 2},
+ {3, 3},
+ {4, 4},
+ {5, 4},
+ {9, 4},
+ {12, 4},
+ {16, 4},
+ {26, 5},
+ {100, 10},
+ }
+ for _, test := range tests {
+ testBroadcastBlock(t, test.totalPeers, test.broadcastExpected)
}
- // Create a DAO aware protocol manager
+}
+
+func testBroadcastBlock(t *testing.T, totalPeers, broadcastExpected int) {
var (
- evmux = new(event.TypeMux)
- pow = ethash.NewFaker()
- db = rawdb.NewMemoryDatabase()
- config = ¶ms.ChainConfig{DAOForkBlock: big.NewInt(1), DAOForkSupport: localForked}
- gspec = &core.Genesis{Config: config}
- genesis = gspec.MustCommit(db)
- blockchain, _ = core.NewBlockChain(db, nil, config, pow, vm.Config{})
+ evmux = new(event.TypeMux)
+ pow = ethash.NewFaker()
+ db = rawdb.NewMemoryDatabase()
+ config = ¶ms.ChainConfig{}
+ gspec = &core.Genesis{Config: config}
+ genesis = gspec.MustCommit(db)
)
- pm, err := NewProtocolManager(config, downloader.FullSync, ethconfig.Defaults.NetworkId, evmux, new(testTxPool), pow, blockchain, db)
+ blockchain, err := core.NewBlockChain(db, nil, config, pow, vm.Config{})
+ if err != nil {
+ t.Fatalf("failed to create new blockchain: %v", err)
+ }
+ pm, err := NewProtocolManager(config, downloader.FullSync, DefaultConfig.NetworkId, evmux, &testTxPool{pool: make(map[common.Hash]*types.Transaction)}, pow, blockchain, db)
if err != nil {
t.Fatalf("failed to start test protocol manager: %v", err)
}
pm.Start(1000)
defer pm.Stop()
-
- // Connect a new peer and check that we receive the DAO challenge
- peer, _ := newTestPeer("peer", eth63, pm, true)
- defer peer.close()
-
- challenge := &getBlockHeadersData{
- Origin: hashOrNumber{Number: config.DAOForkBlock.Uint64()},
- Amount: 1,
- Skip: 0,
- Reverse: false,
+ var peers []*testPeer
+ for i := 0; i < totalPeers; i++ {
+ peer, _ := newTestPeer(fmt.Sprintf("peer %d", i), eth63, pm, true)
+ defer peer.close()
+ peers = append(peers, peer)
}
- if err := p2p.ExpectMsg(peer.app, GetBlockHeadersMsg, challenge); err != nil {
- t.Fatalf("challenge mismatch: %v", err)
+ chain, _ := core.GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, 1, func(i int, gen *core.BlockGen) {})
+ pm.BroadcastBlock(chain[0], true /*propagate*/)
+
+ errCh := make(chan error, totalPeers)
+ doneCh := make(chan struct{}, totalPeers)
+ for _, peer := range peers {
+ go func(p *testPeer) {
+ if err := p2p.ExpectMsg(p.app, NewBlockMsg, &newBlockData{Block: chain[0], TD: big.NewInt(131136)}); err != nil {
+ errCh <- err
+ } else {
+ doneCh <- struct{}{}
+ }
+ }(peer)
}
- // Create a block to reply to the challenge if no timeout is simulated
- if !timeout {
- blocks, _ := core.GenerateChain(¶ms.ChainConfig{}, genesis, ethash.NewFaker(), db, 1, func(i int, block *core.BlockGen) {
- if remoteForked {
- block.SetExtra(params.DAOForkBlockExtra)
+ timeout := time.After(2 * time.Second)
+ var receivedCount int
+outer:
+ for {
+ select {
+ case err = <-errCh:
+ break outer
+ case <-doneCh:
+ receivedCount++
+ if receivedCount == totalPeers {
+ break outer
}
- })
- if err := p2p.Send(peer.app, BlockHeadersMsg, []*types.Header{blocks[0].Header()}); err != nil {
- t.Fatalf("failed to answer challenge: %v", err)
+ case <-timeout:
+ break outer
}
- time.Sleep(100 * time.Millisecond) // Sleep to avoid the verification racing with the drops
- } else {
- // Otherwise wait until the test timeout passes
- time.Sleep(daoChallengeTimeout + 500*time.Millisecond)
}
- // Verify that depending on fork side, the remote peer is maintained or dropped
- if localForked == remoteForked && !timeout {
- if peers := pm.peers.Len(); peers != 1 {
- t.Fatalf("peer count mismatch: have %d, want %d", peers, 1)
+ for _, peer := range peers {
+ peer.app.Close()
+ }
+ if err != nil {
+ t.Errorf("error matching block by peer: %v", err)
+ }
+ if receivedCount != broadcastExpected {
+ t.Errorf("block broadcast to %d peers, expected %d", receivedCount, broadcastExpected)
+ }
+}
+
+// Tests that a propagated malformed block (uncles or transactions don't match
+// with the hashes in the header) gets discarded and not broadcast forward.
+func TestBroadcastMalformedBlock(t *testing.T) {
+ // Create a live node to test propagation with
+ var (
+ engine = ethash.NewFaker()
+ db = rawdb.NewMemoryDatabase()
+ config = ¶ms.ChainConfig{}
+ gspec = &core.Genesis{Config: config}
+ genesis = gspec.MustCommit(db)
+ )
+ blockchain, err := core.NewBlockChain(db, nil, config, engine, vm.Config{})
+ if err != nil {
+ t.Fatalf("failed to create new blockchain: %v", err)
+ }
+ pm, err := NewProtocolManager(config, downloader.FullSync, DefaultConfig.NetworkId, new(event.TypeMux), new(testTxPool), engine, blockchain, db)
+ if err != nil {
+ t.Fatalf("failed to start test protocol manager: %v", err)
+ }
+ pm.Start(2)
+ defer pm.Stop()
+
+ // Create two peers, one to send the malformed block with and one to check
+ // propagation
+ source, _ := newTestPeer("source", eth63, pm, true)
+ defer source.close()
+
+ sink, _ := newTestPeer("sink", eth63, pm, true)
+ defer sink.close()
+
+ // Create various combinations of malformed blocks
+ chain, _ := core.GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, 1, func(i int, gen *core.BlockGen) {})
+
+ malformedUncles := chain[0].Header()
+ malformedUncles.UncleHash[0]++
+ malformedTransactions := chain[0].Header()
+ malformedTransactions.TxHash[0]++
+ malformedEverything := chain[0].Header()
+ malformedEverything.UncleHash[0]++
+ malformedEverything.TxHash[0]++
+
+ // Keep listening to broadcasts and notify if any arrives
+ notify := make(chan struct{})
+ go func() {
+ if _, err := sink.app.ReadMsg(); err == nil {
+ notify <- struct{}{}
+ }
+ }()
+ // Try to broadcast all malformations and ensure they all get discarded
+ for _, header := range []*types.Header{malformedUncles, malformedTransactions, malformedEverything} {
+ block := types.NewBlockWithHeader(header).WithBody(chain[0].Transactions(), chain[0].Uncles())
+ if err := p2p.Send(source.app, NewBlockMsg, []interface{}{block, big.NewInt(131136)}); err != nil {
+ t.Fatalf("failed to broadcast block: %v", err)
}
- } else {
- if peers := pm.peers.Len(); peers != 0 {
- t.Fatalf("peer count mismatch: have %d, want %d", peers, 0)
+ select {
+ case <-notify:
+ t.Fatalf("malformed block forwarded")
+ case <-time.After(100 * time.Millisecond):
}
}
}
diff --git a/eth/helper_test.go b/eth/helper_test.go
index aa09d9edb398..3f25658586a1 100644
--- a/eth/helper_test.go
+++ b/eth/helper_test.go
@@ -70,7 +70,8 @@ func newTestProtocolManager(mode downloader.SyncMode, blocks int, generator func
panic(err)
}
- pm, err := NewProtocolManager(gspec.Config, mode, ethconfig.Defaults.NetworkId, evmux, &testTxPool{added: newtx}, engine, blockchain, db)
+ // pm, err := NewProtocolManager(gspec.Config, mode, DefaultConfig.NetworkId, evmux, &testTxPool{added: newtx}, engine, blockchain, db)
+ pm, err := NewProtocolManager(gspec.Config, mode, DefaultConfig.NetworkId, evmux, &testTxPool{added: newtx, pool: make(map[common.Hash]*types.Transaction)}, engine, blockchain, db)
if err != nil {
return nil, nil, err
}
@@ -93,22 +94,43 @@ func newTestProtocolManagerMust(t *testing.T, mode downloader.SyncMode, blocks i
// testTxPool is a fake, helper transaction pool for testing purposes
type testTxPool struct {
txFeed event.Feed
- pool []*types.Transaction // Collection of all transactions
- added chan<- []*types.Transaction // Notification channel for new transactions
+ pool map[common.Hash]*types.Transaction // Hash map of collected transactions
+ added chan<- []*types.Transaction // Notification channel for new transactions
lock sync.RWMutex // Protects the transaction pool
}
+// Has returns an indicator whether txpool has a transaction
+// cached with the given hash.
+func (p *testTxPool) Has(hash common.Hash) bool {
+ p.lock.Lock()
+ defer p.lock.Unlock()
+
+ return p.pool[hash] != nil
+}
+
+// Get retrieves the transaction from local txpool with given
+// tx hash.
+func (p *testTxPool) Get(hash common.Hash) *types.Transaction {
+ p.lock.Lock()
+ defer p.lock.Unlock()
+
+ return p.pool[hash]
+}
+
// AddRemotes appends a batch of transactions to the pool, and notifies any
// listeners if the addition channel is non nil
func (p *testTxPool) AddRemotes(txs []*types.Transaction) []error {
p.lock.Lock()
defer p.lock.Unlock()
- p.pool = append(p.pool, txs...)
+ for _, tx := range txs {
+ p.pool[tx.Hash()] = tx
+ }
if p.added != nil {
p.added <- txs
}
+ p.txFeed.Send(core.NewTxsEvent{Txs: txs})
return make([]error, len(txs))
}
@@ -155,7 +177,7 @@ func newTestPeer(name string, version int, pm *ProtocolManager, shake bool) (*te
var id enode.ID
rand.Read(id[:])
- peer := pm.newPeer(version, p2p.NewPeer(id, name, nil), net)
+ peer := pm.newPeer(version, p2p.NewPeer(id, name, nil), net, pm.txpool.Get)
// Start the peer on a new thread
errc := make(chan error, 1)
@@ -193,7 +215,7 @@ func (p *testPeer) handshake(t *testing.T, td *big.Int, head common.Hash, genesi
CurrentBlock: head,
GenesisBlock: genesis,
}
- case p.version == eth64:
+ case p.version >= eth64:
msg = &statusData{
ProtocolVersion: uint32(p.version),
NetworkID: DefaultConfig.NetworkId,
diff --git a/eth/metrics.go b/eth/metrics.go
deleted file mode 100644
index 2f3bd6bfb839..000000000000
--- a/eth/metrics.go
+++ /dev/null
@@ -1,139 +0,0 @@
-// Copyright 2015 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see .
-
-package eth
-
-import (
- "github.com/XinFinOrg/XDPoSChain/metrics"
- "github.com/XinFinOrg/XDPoSChain/p2p"
-)
-
-var (
- propTxnInPacketsMeter = metrics.NewRegisteredMeter("eth/prop/txns/in/packets", nil)
- propTxnInTrafficMeter = metrics.NewRegisteredMeter("eth/prop/txns/in/traffic", nil)
- propTxnOutPacketsMeter = metrics.NewRegisteredMeter("eth/prop/txns/out/packets", nil)
- propTxnOutTrafficMeter = metrics.NewRegisteredMeter("eth/prop/txns/out/traffic", nil)
- propHashInPacketsMeter = metrics.NewRegisteredMeter("eth/prop/hashes/in/packets", nil)
- propHashInTrafficMeter = metrics.NewRegisteredMeter("eth/prop/hashes/in/traffic", nil)
- propHashOutPacketsMeter = metrics.NewRegisteredMeter("eth/prop/hashes/out/packets", nil)
- propHashOutTrafficMeter = metrics.NewRegisteredMeter("eth/prop/hashes/out/traffic", nil)
- propBlockInPacketsMeter = metrics.NewRegisteredMeter("eth/prop/blocks/in/packets", nil)
- propBlockInTrafficMeter = metrics.NewRegisteredMeter("eth/prop/blocks/in/traffic", nil)
- propBlockOutPacketsMeter = metrics.NewRegisteredMeter("eth/prop/blocks/out/packets", nil)
- propBlockOutTrafficMeter = metrics.NewRegisteredMeter("eth/prop/blocks/out/traffic", nil)
- reqHeaderInPacketsMeter = metrics.NewRegisteredMeter("eth/req/headers/in/packets", nil)
- reqHeaderInTrafficMeter = metrics.NewRegisteredMeter("eth/req/headers/in/traffic", nil)
- reqHeaderOutPacketsMeter = metrics.NewRegisteredMeter("eth/req/headers/out/packets", nil)
- reqHeaderOutTrafficMeter = metrics.NewRegisteredMeter("eth/req/headers/out/traffic", nil)
- reqBodyInPacketsMeter = metrics.NewRegisteredMeter("eth/req/bodies/in/packets", nil)
- reqBodyInTrafficMeter = metrics.NewRegisteredMeter("eth/req/bodies/in/traffic", nil)
- reqBodyOutPacketsMeter = metrics.NewRegisteredMeter("eth/req/bodies/out/packets", nil)
- reqBodyOutTrafficMeter = metrics.NewRegisteredMeter("eth/req/bodies/out/traffic", nil)
- reqStateInPacketsMeter = metrics.NewRegisteredMeter("eth/req/states/in/packets", nil)
- reqStateInTrafficMeter = metrics.NewRegisteredMeter("eth/req/states/in/traffic", nil)
- reqStateOutPacketsMeter = metrics.NewRegisteredMeter("eth/req/states/out/packets", nil)
- reqStateOutTrafficMeter = metrics.NewRegisteredMeter("eth/req/states/out/traffic", nil)
- reqReceiptInPacketsMeter = metrics.NewRegisteredMeter("eth/req/receipts/in/packets", nil)
- reqReceiptInTrafficMeter = metrics.NewRegisteredMeter("eth/req/receipts/in/traffic", nil)
- reqReceiptOutPacketsMeter = metrics.NewRegisteredMeter("eth/req/receipts/out/packets", nil)
- reqReceiptOutTrafficMeter = metrics.NewRegisteredMeter("eth/req/receipts/out/traffic", nil)
- miscInPacketsMeter = metrics.NewRegisteredMeter("eth/misc/in/packets", nil)
- miscInTrafficMeter = metrics.NewRegisteredMeter("eth/misc/in/traffic", nil)
- miscOutPacketsMeter = metrics.NewRegisteredMeter("eth/misc/out/packets", nil)
- miscOutTrafficMeter = metrics.NewRegisteredMeter("eth/misc/out/traffic", nil)
-)
-
-// meteredMsgReadWriter is a wrapper around a p2p.MsgReadWriter, capable of
-// accumulating the above defined metrics based on the data stream contents.
-type meteredMsgReadWriter struct {
- p2p.MsgReadWriter // Wrapped message stream to meter
- version int // Protocol version to select correct meters
-}
-
-// newMeteredMsgWriter wraps a p2p MsgReadWriter with metering support. If the
-// metrics system is disabled, this function returns the original object.
-func newMeteredMsgWriter(rw p2p.MsgReadWriter) p2p.MsgReadWriter {
- if !metrics.Enabled {
- return rw
- }
- return &meteredMsgReadWriter{MsgReadWriter: rw}
-}
-
-// Init sets the protocol version used by the stream to know which meters to
-// increment in case of overlapping message ids between protocol versions.
-func (rw *meteredMsgReadWriter) Init(version int) {
- rw.version = version
-}
-
-func (rw *meteredMsgReadWriter) ReadMsg() (p2p.Msg, error) {
- // Read the message and short circuit in case of an error
- msg, err := rw.MsgReadWriter.ReadMsg()
- if err != nil {
- return msg, err
- }
- // Account for the data traffic
- packets, traffic := miscInPacketsMeter, miscInTrafficMeter
- switch {
- case msg.Code == BlockHeadersMsg:
- packets, traffic = reqHeaderInPacketsMeter, reqHeaderInTrafficMeter
- case msg.Code == BlockBodiesMsg:
- packets, traffic = reqBodyInPacketsMeter, reqBodyInTrafficMeter
-
- case rw.version >= eth63 && msg.Code == NodeDataMsg:
- packets, traffic = reqStateInPacketsMeter, reqStateInTrafficMeter
- case rw.version >= eth63 && msg.Code == ReceiptsMsg:
- packets, traffic = reqReceiptInPacketsMeter, reqReceiptInTrafficMeter
-
- case msg.Code == NewBlockHashesMsg:
- packets, traffic = propHashInPacketsMeter, propHashInTrafficMeter
- case msg.Code == NewBlockMsg:
- packets, traffic = propBlockInPacketsMeter, propBlockInTrafficMeter
- case msg.Code == TxMsg:
- packets, traffic = propTxnInPacketsMeter, propTxnInTrafficMeter
- }
- packets.Mark(1)
- traffic.Mark(int64(msg.Size))
-
- return msg, err
-}
-
-func (rw *meteredMsgReadWriter) WriteMsg(msg p2p.Msg) error {
- // Account for the data traffic
- packets, traffic := miscOutPacketsMeter, miscOutTrafficMeter
- switch {
- case msg.Code == BlockHeadersMsg:
- packets, traffic = reqHeaderOutPacketsMeter, reqHeaderOutTrafficMeter
- case msg.Code == BlockBodiesMsg:
- packets, traffic = reqBodyOutPacketsMeter, reqBodyOutTrafficMeter
-
- case rw.version >= eth63 && msg.Code == NodeDataMsg:
- packets, traffic = reqStateOutPacketsMeter, reqStateOutTrafficMeter
- case rw.version >= eth63 && msg.Code == ReceiptsMsg:
- packets, traffic = reqReceiptOutPacketsMeter, reqReceiptOutTrafficMeter
-
- case msg.Code == NewBlockHashesMsg:
- packets, traffic = propHashOutPacketsMeter, propHashOutTrafficMeter
- case msg.Code == NewBlockMsg:
- packets, traffic = propBlockOutPacketsMeter, propBlockOutTrafficMeter
- case msg.Code == TxMsg:
- packets, traffic = propTxnOutPacketsMeter, propTxnOutTrafficMeter
- }
- packets.Mark(1)
- traffic.Mark(int64(msg.Size))
-
- // Send the packet to the p2p layer
- return rw.MsgReadWriter.WriteMsg(msg)
-}
diff --git a/eth/peer.go b/eth/peer.go
index 46f5e05aeffa..5e1a7a29ec4b 100644
--- a/eth/peer.go
+++ b/eth/peer.go
@@ -45,9 +45,38 @@ const (
maxKnownVote = 1024 // Maximum transactions hashes to keep in the known list (prevent DOS)
maxKnownTimeout = 1024 // Maximum transactions hashes to keep in the known list (prevent DOS)
maxKnownSyncInfo = 1024 // Maximum transactions hashes to keep in the known list (prevent DOS)
- handshakeTimeout = 5 * time.Second
+ // maxQueuedTxs is the maximum number of transactions to queue up before dropping
+ // older broadcasts.
+ maxQueuedTxs = 4096
+ // maxQueuedTxAnns is the maximum number of transaction announcements to queue up
+ // before dropping older announcements.
+ maxQueuedTxAnns = 4096
+ // maxQueuedBlocks is the maximum number of block propagations to queue up before
+ // dropping broadcasts. There's not much point in queueing stale blocks, so a few
+ // that might cover uncles should be enough.
+ maxQueuedBlocks = 4
+ // maxQueuedBlockAnns is the maximum number of block announcements to queue up before
+ // dropping broadcasts. Similarly to block propagations, there's no point to queue
+ // above some healthy uncle limit, so use that.
+ maxQueuedBlockAnns = 4
+
+ handshakeTimeout = 5 * time.Second
)
+// max is a helper function which returns the larger of the two given integers.
+func max(a, b int) int {
+ if a > b {
+ return a
+ }
+ return b
+}
+
+// propEvent is a block propagation, waiting for its turn in the broadcast queue.
+type propEvent struct {
+ block *types.Block
+ td *big.Int
+}
+
// PeerInfo represents a short summary of the Ethereum sub-protocol metadata known
// about a connected peer.
type PeerInfo struct {
@@ -70,36 +99,199 @@ type peer struct {
td *big.Int
lock sync.RWMutex
- knownTxs mapset.Set // Set of transaction hashes known to be known by this peer
- knownBlocks mapset.Set // Set of block hashes known to be known by this peer
-
+ knownBlocks mapset.Set // Set of block hashes known to be known by this peer
+ knownTxs mapset.Set // Set of transaction hashes known to be known by this peer
knownOrderTxs mapset.Set // Set of order transaction hashes known to be known by this peer
knownLendingTxs mapset.Set // Set of lending transaction hashes known to be known by this peer
+ knownVote mapset.Set // Set of BFT Vote known to be known by this peer
+ knownTimeout mapset.Set // Set of BFT timeout known to be known by this peer
+ knownSyncInfo mapset.Set // Set of BFT Sync Info known to be known by this peer
- knownVote mapset.Set // Set of BFT Vote known to be known by this peer
- knownTimeout mapset.Set // Set of BFT timeout known to be known by this peer
- knownSyncInfo mapset.Set // Set of BFT Sync Info known to be known by this peer`
-}
+ queuedBlocks chan *propEvent // Queue of blocks to broadcast to the peer
+ queuedBlockAnns chan *types.Block // Queue of blocks to announce to the peer
+
+ txBroadcast chan []common.Hash // Channel used to queue transaction propagation requests
+ txAnnounce chan []common.Hash // Channel used to queue transaction announcement requests
+ getPooledTx func(common.Hash) *types.Transaction // Callback used to retrieve transaction from txpool
-func newPeer(version int, p *p2p.Peer, rw p2p.MsgReadWriter) *peer {
- id := p.ID()
+ term chan struct{} // Termination channel to stop the broadcaster
+}
+func newPeer(version int, p *p2p.Peer, rw p2p.MsgReadWriter, getPooledTx func(hash common.Hash) *types.Transaction) *peer {
return &peer{
Peer: p,
rw: rw,
version: version,
- id: fmt.Sprintf("%x", id[:8]),
+ id: fmt.Sprintf("%x", p.ID().Bytes()[:8]),
knownTxs: mapset.NewSet(),
knownBlocks: mapset.NewSet(),
knownOrderTxs: mapset.NewSet(),
knownLendingTxs: mapset.NewSet(),
+ knownVote: mapset.NewSet(),
+ knownTimeout: mapset.NewSet(),
+ knownSyncInfo: mapset.NewSet(),
+ queuedBlocks: make(chan *propEvent, maxQueuedBlocks),
+ queuedBlockAnns: make(chan *types.Block, maxQueuedBlockAnns),
+ txBroadcast: make(chan []common.Hash),
+ txAnnounce: make(chan []common.Hash),
+ getPooledTx: getPooledTx,
+ term: make(chan struct{}),
+ }
+}
+
+// broadcastBlocks is a write loop that multiplexes blocks and block accouncements
+// to the remote peer. The goal is to have an async writer that does not lock up
+// node internals and at the same time rate limits queued data.
+func (p *peer) broadcastBlocks() {
+ for {
+ select {
+ case prop := <-p.queuedBlocks:
+ if err := p.SendNewBlock(prop.block, prop.td); err != nil {
+ return
+ }
+ p.Log().Trace("Propagated block", "number", prop.block.Number(), "hash", prop.block.Hash(), "td", prop.td)
+
+ case block := <-p.queuedBlockAnns:
+ if err := p.SendNewBlockHashes([]common.Hash{block.Hash()}, []uint64{block.NumberU64()}); err != nil {
+ return
+ }
+ p.Log().Trace("Announced block", "number", block.Number(), "hash", block.Hash())
+
+ case <-p.term:
+ return
+ }
+ }
+}
+
+// broadcastTransactions is a write loop that schedules transaction broadcasts
+// to the remote peer. The goal is to have an async writer that does not lock up
+// node internals and at the same time rate limits queued data.
+func (p *peer) broadcastTransactions() {
+ var (
+ queue []common.Hash // Queue of hashes to broadcast as full transactions
+ done chan struct{} // Non-nil if background broadcaster is running
+ fail = make(chan error) // Channel used to receive network error
+ )
+ for {
+ // If there's no in-flight broadcast running, check if a new one is needed
+ if done == nil && len(queue) > 0 {
+ // Pile transaction until we reach our allowed network limit
+ var (
+ hashes []common.Hash
+ txs []*types.Transaction
+ size common.StorageSize
+ )
+ for i := 0; i < len(queue) && size < txsyncPackSize; i++ {
+ if tx := p.getPooledTx(queue[i]); tx != nil {
+ txs = append(txs, tx)
+ size += tx.Size()
+ }
+ hashes = append(hashes, queue[i])
+ }
+ queue = queue[:copy(queue, queue[len(hashes):])]
+
+ // If there's anything available to transfer, fire up an async writer
+ if len(txs) > 0 {
+ done = make(chan struct{})
+ go func() {
+ if err := p.sendTransactions(txs); err != nil {
+ fail <- err
+ return
+ }
+ close(done)
+ p.Log().Trace("Sent transactions", "count", len(txs))
+ }()
+ }
+ }
+ // Transfer goroutine may or may not have been started, listen for events
+ select {
+ case hashes := <-p.txBroadcast:
+ // New batch of transactions to be broadcast, queue them (with cap)
+ queue = append(queue, hashes...)
+ if len(queue) > maxQueuedTxs {
+ // Fancy copy and resize to ensure buffer doesn't grow indefinitely
+ queue = queue[:copy(queue, queue[len(queue)-maxQueuedTxs:])]
+ }
+
+ case <-done:
+ done = nil
+
+ case <-fail:
+ return
+
+ case <-p.term:
+ return
+ }
+ }
+}
+
+// announceTransactions is a write loop that schedules transaction broadcasts
+// to the remote peer. The goal is to have an async writer that does not lock up
+// node internals and at the same time rate limits queued data.
+func (p *peer) announceTransactions() {
+ var (
+ queue []common.Hash // Queue of hashes to announce as transaction stubs
+ done chan struct{} // Non-nil if background announcer is running
+ fail = make(chan error) // Channel used to receive network error
+ )
+ for {
+ // If there's no in-flight announce running, check if a new one is needed
+ if done == nil && len(queue) > 0 {
+ // Pile transaction hashes until we reach our allowed network limit
+ var (
+ hashes []common.Hash
+ pending []common.Hash
+ size common.StorageSize
+ )
+ for i := 0; i < len(queue) && size < txsyncPackSize; i++ {
+ if p.getPooledTx(queue[i]) != nil {
+ pending = append(pending, queue[i])
+ size += common.HashLength
+ }
+ hashes = append(hashes, queue[i])
+ }
+ queue = queue[:copy(queue, queue[len(hashes):])]
+
+ // If there's anything available to transfer, fire up an async writer
+ if len(pending) > 0 {
+ done = make(chan struct{})
+ go func() {
+ if err := p.sendPooledTransactionHashes(pending); err != nil {
+ fail <- err
+ return
+ }
+ close(done)
+ p.Log().Trace("Sent transaction announcements", "count", len(pending))
+ }()
+ }
+ }
+ // Transfer goroutine may or may not have been started, listen for events
+ select {
+ case hashes := <-p.txAnnounce:
+ // New batch of transactions to be broadcast, queue them (with cap)
+ queue = append(queue, hashes...)
+ if len(queue) > maxQueuedTxAnns {
+ // Fancy copy and resize to ensure buffer doesn't grow indefinitely
+ queue = queue[:copy(queue, queue[len(queue)-maxQueuedTxs:])]
+ }
- knownVote: mapset.NewSet(),
- knownTimeout: mapset.NewSet(),
- knownSyncInfo: mapset.NewSet(),
+ case <-done:
+ done = nil
+
+ case <-fail:
+ return
+
+ case <-p.term:
+ return
+ }
}
}
+// close signals the broadcast goroutine to terminate.
+func (p *peer) close() {
+ close(p.term)
+}
+
// Info gathers and returns a collection of metadata known about a peer.
func (p *peer) Info() *PeerInfo {
hash, td := p.Head()
@@ -200,16 +392,41 @@ func (p *peer) MarkSyncInfo(hash common.Hash) {
p.knownSyncInfo.Add(hash)
}
-// SendTransactions sends transactions to the peer and includes the hashes
+// SendTransactions64 sends transactions to the peer and includes the hashes
// in its transaction hash set for future reference.
-func (p *peer) SendTransactions(txs types.Transactions) error {
- for p.knownTxs.Cardinality() >= maxKnownTxs {
+//
+// This method is legacy support for initial transaction exchange in eth/64 and
+// prior. For eth/65 and higher use SendPooledTransactionHashes.
+func (p *peer) SendTransactions64(txs types.Transactions) error {
+ return p.sendTransactions(txs)
+}
+
+// // SendTransactions sends transactions to the peer and includes the hashes
+// // in its transaction hash set for future reference.
+// func (p *peer) SendTransactions(txs types.Transactions) error {
+// for p.knownTxs.Cardinality() >= maxKnownTxs {
+// p.knownTxs.Pop()
+// }
+// for _, tx := range txs {
+// p.knownTxs.Add(tx.Hash())
+// return p2p.Send(p.rw, TxMsg, txs)
+// }
+
+// sendTransactions sends transactions to the peer and includes the hashes
+// in its transaction hash set for future reference.
+//
+// This method is a helper used by the async transaction sender. Don't call it
+// directly as the queueing (memory) and transmission (bandwidth) costs should
+// not be managed directly.
+func (p *peer) sendTransactions(txs types.Transactions) error {
+ // Mark all the transactions as known, but ensure we don't overflow our limits
+ for p.knownTxs.Cardinality() > max(0, maxKnownTxs-len(txs)) {
p.knownTxs.Pop()
}
for _, tx := range txs {
p.knownTxs.Add(tx.Hash())
}
- return p2p.Send(p.rw, TxMsg, txs)
+ return p2p.Send(p.rw, TransactionMsg, txs)
}
// SendTransactions sends transactions to the peer and includes the hashes
@@ -225,6 +442,24 @@ func (p *peer) SendOrderTransactions(txs types.OrderTransactions) error {
return p2p.Send(p.rw, OrderTxMsg, txs)
}
+// AsyncSendTransactions queues a list of transactions (by hash) to eventually
+// propagate to a remote peer. The number of pending sends are capped (new ones
+// will force old sends to be dropped)
+func (p *peer) AsyncSendTransactions(hashes []common.Hash) {
+ select {
+ case p.txBroadcast <- hashes:
+ // Mark all the transactions as known, but ensure we don't overflow our limits
+ for p.knownTxs.Cardinality() > max(0, maxKnownTxs-len(hashes)) {
+ p.knownTxs.Pop()
+ }
+ for _, hash := range hashes {
+ p.knownTxs.Add(hash)
+ }
+ case <-p.term:
+ p.Log().Debug("Dropping transaction propagation", "count", len(hashes))
+ }
+}
+
// SendTransactions sends transactions to the peer and includes the hashes
// in its transaction hash set for future reference.
func (p *peer) SendLendingTransactions(txs types.LendingTransactions) error {
@@ -238,13 +473,64 @@ func (p *peer) SendLendingTransactions(txs types.LendingTransactions) error {
return p2p.Send(p.rw, LendingTxMsg, txs)
}
+// sendPooledTransactionHashes sends transaction hashes to the peer and includes
+// them in its transaction hash set for future reference.
+//
+// This method is a helper used by the async transaction announcer. Don't call it
+// directly as the queueing (memory) and transmission (bandwidth) costs should
+// not be managed directly.
+func (p *peer) sendPooledTransactionHashes(hashes []common.Hash) error {
+ // Mark all the transactions as known, but ensure we don't overflow our limits
+ for p.knownTxs.Cardinality() > max(0, maxKnownTxs-len(hashes)) {
+ p.knownTxs.Pop()
+ }
+ for _, hash := range hashes {
+ p.knownTxs.Add(hash)
+ }
+ return p2p.Send(p.rw, NewPooledTransactionHashesMsg, hashes)
+}
+
+// AsyncSendPooledTransactionHashes queues a list of transactions hashes to eventually
+// announce to a remote peer. The number of pending sends are capped (new ones
+// will force old sends to be dropped)
+func (p *peer) AsyncSendPooledTransactionHashes(hashes []common.Hash) {
+ select {
+ case p.txAnnounce <- hashes:
+ // Mark all the transactions as known, but ensure we don't overflow our limits
+ for p.knownTxs.Cardinality() > max(0, maxKnownTxs-len(hashes)) {
+ p.knownTxs.Pop()
+ }
+ for _, hash := range hashes {
+ p.knownTxs.Add(hash)
+ }
+ case <-p.term:
+ p.Log().Debug("Dropping transaction announcement", "count", len(hashes))
+ }
+}
+
+// SendPooledTransactionsRLP sends requested transactions to the peer and adds the
+// hashes in its transaction hash set for future reference.
+//
+// Note, the method assumes the hashes are correct and correspond to the list of
+// transactions being sent.
+func (p *peer) SendPooledTransactionsRLP(hashes []common.Hash, txs []rlp.RawValue) error {
+ // Mark all the transactions as known, but ensure we don't overflow our limits
+ for p.knownTxs.Cardinality() > max(0, maxKnownTxs-len(hashes)) {
+ p.knownTxs.Pop()
+ }
+ for _, hash := range hashes {
+ p.knownTxs.Add(hash)
+ }
+ return p2p.Send(p.rw, PooledTransactionsMsg, txs)
+}
+
// SendNewBlockHashes announces the availability of a number of blocks through
// a hash notification.
func (p *peer) SendNewBlockHashes(hashes []common.Hash, numbers []uint64) error {
- for p.knownBlocks.Cardinality() >= maxKnownBlocks {
+ // Mark all the block hashes as known, but ensure we don't overflow our limits
+ for p.knownBlocks.Cardinality() > max(0, maxKnownBlocks-len(hashes)) {
p.knownBlocks.Pop()
}
-
for _, hash := range hashes {
p.knownBlocks.Add(hash)
}
@@ -256,6 +542,16 @@ func (p *peer) SendNewBlockHashes(hashes []common.Hash, numbers []uint64) error
return p2p.Send(p.rw, NewBlockHashesMsg, request)
}
+// // SendNewBlock propagates an entire block to a remote peer.
+// func (p *peer) SendNewBlock(block *types.Block, td *big.Int) error {
+// // Mark all the block hash as known, but ensure we don't overflow our limits
+// for p.knownBlocks.Cardinality() >= maxKnownBlocks {
+// p.knownBlocks.Pop()
+// }
+// p.knownBlocks.Add(block.Hash())
+// return p2p.Send(p.rw, NewBlockMsg, []interface{}{block, td})
+// }
+
// SendNewBlock propagates an entire block to a remote peer.
func (p *peer) SendNewBlock(block *types.Block, td *big.Int) error {
for p.knownBlocks.Cardinality() >= maxKnownBlocks {
@@ -270,6 +566,37 @@ func (p *peer) SendNewBlock(block *types.Block, td *big.Int) error {
}
}
+// AsyncSendNewBlockHash queues the availability of a block for propagation to a
+// remote peer. If the peer's broadcast queue is full, the event is silently
+// dropped.
+func (p *peer) AsyncSendNewBlockHash(block *types.Block) {
+ select {
+ case p.queuedBlockAnns <- block:
+ // Mark all the block hash as known, but ensure we don't overflow our limits
+ for p.knownBlocks.Cardinality() >= maxKnownBlocks {
+ p.knownBlocks.Pop()
+ }
+ p.knownBlocks.Add(block.Hash())
+ default:
+ p.Log().Debug("Dropping block announcement", "number", block.NumberU64(), "hash", block.Hash())
+ }
+}
+
+// AsyncSendNewBlock queues an entire block for propagation to a remote peer. If
+// the peer's broadcast queue is full, the event is silently dropped.
+func (p *peer) AsyncSendNewBlock(block *types.Block, td *big.Int) {
+ select {
+ case p.queuedBlocks <- &propEvent{block: block, td: td}:
+ // Mark all the block hash as known, but ensure we don't overflow our limits
+ for p.knownBlocks.Cardinality() >= maxKnownBlocks {
+ p.knownBlocks.Pop()
+ }
+ p.knownBlocks.Add(block.Hash())
+ default:
+ p.Log().Debug("Dropping block propagation", "number", block.NumberU64(), "hash", block.Hash())
+ }
+}
+
// SendBlockHeaders sends a batch of block headers to the remote peer.
func (p *peer) SendBlockHeaders(headers []*types.Header) error {
if p.pairRw != nil {
@@ -438,6 +765,12 @@ func (p *peer) RequestReceipts(hashes []common.Hash) error {
}
}
+// RequestTxs fetches a batch of transactions from a remote node.
+func (p *peer) RequestTxs(hashes []common.Hash) error {
+ p.Log().Debug("Fetching batch of transactions", "count", len(hashes))
+ return p2p.Send(p.rw, GetPooledTransactionsMsg, hashes)
+}
+
// Handshake executes the eth protocol handshake, negotiating version number,
// network IDs, difficulties, head and genesis blocks.
func (p *peer) Handshake(network uint64, td *big.Int, head common.Hash, genesis common.Hash, forkID forkid.ID, forkFilter forkid.Filter) error {
@@ -458,7 +791,7 @@ func (p *peer) Handshake(network uint64, td *big.Int, head common.Hash, genesis
CurrentBlock: head,
GenesisBlock: genesis,
})
- case p.version == eth64:
+ case p.version >= eth64:
errc <- p2p.Send(p.rw, StatusMsg, &statusData{
ProtocolVersion: uint32(p.version),
NetworkID: network,
@@ -475,7 +808,7 @@ func (p *peer) Handshake(network uint64, td *big.Int, head common.Hash, genesis
switch {
case p.version == eth63:
errc <- p.readStatusLegacy(network, &status63, genesis)
- case p.version == eth64:
+ case p.version >= eth64:
errc <- p.readStatus(network, &status, genesis, forkFilter)
default:
panic(fmt.Sprintf("unsupported eth protocol version: %d", p.version))
@@ -496,7 +829,7 @@ func (p *peer) Handshake(network uint64, td *big.Int, head common.Hash, genesis
switch {
case p.version == eth63:
p.td, p.head = status63.TD, status63.CurrentBlock
- case p.version == eth64:
+ case p.version >= eth64:
p.td, p.head = status.TD, status.Head
default:
panic(fmt.Sprintf("unsupported eth protocol version: %d", p.version))
@@ -602,6 +935,11 @@ func (ps *peerSet) Register(p *peer) error {
return p2p.ErrAddPairPeer
}
ps.peers[p.id] = p
+
+ go p.broadcastBlocks()
+ go p.broadcastTransactions()
+ go p.announceTransactions()
+
return nil
}
diff --git a/eth/protocol.go b/eth/protocol.go
index ec8caf30938e..819db3c857a9 100644
--- a/eth/protocol.go
+++ b/eth/protocol.go
@@ -33,6 +33,7 @@ import (
const (
eth63 = 63
eth64 = 64
+ eth65 = 65
xdpos2 = 100
)
@@ -40,10 +41,10 @@ const (
const protocolName = "eth"
// ProtocolVersions are the supported versions of the eth protocol (first is primary).
-var ProtocolVersions = []uint{xdpos2, eth64, eth63}
+var ProtocolVersions = []uint{xdpos2, eth65, eth64, eth63}
// protocolLengths are the number of implemented message corresponding to different protocol versions.
-var protocolLengths = map[uint]uint64{xdpos2: 227, eth64: 17, eth63: 17}
+var protocolLengths = map[uint]uint64{xdpos2: 227, eth65: 17, eth64: 17, eth63: 17}
const protocolMaxMsgSize = 10 * 1024 * 1024 // Maximum cap on the size of a protocol message
@@ -51,7 +52,7 @@ const protocolMaxMsgSize = 10 * 1024 * 1024 // Maximum cap on the size of a prot
const (
StatusMsg = 0x00
NewBlockHashesMsg = 0x01
- TxMsg = 0x02
+ TransactionMsg = 0x02
GetBlockHeadersMsg = 0x03
BlockHeadersMsg = 0x04
GetBlockBodiesMsg = 0x05
@@ -65,6 +66,14 @@ const (
GetReceiptsMsg = 0x0f
ReceiptsMsg = 0x10
+ // New protocol message codes introduced in eth65
+ //
+ // Previously these message ids were used by some legacy and unsupported
+ // eth protocols, reown them here.
+ NewPooledTransactionHashesMsg = 0x28 //originally 0x08 but clash with OrderTxMsg
+ GetPooledTransactionsMsg = 0x29 //originally 0x09 but clash with LendingTxMsg
+ PooledTransactionsMsg = 0x0a
+
// Protocol messages belonging to xdpos2/100
VoteMsg = 0xe0
TimeoutMsg = 0xe1
@@ -103,6 +112,14 @@ var errorToString = map[int]string{
}
type txPool interface {
+ // Has returns an indicator whether txpool has a transaction
+ // cached with the given hash.
+ Has(hash common.Hash) bool
+
+ // Get retrieves the transaction from local txpool with given
+ // tx hash.
+ Get(hash common.Hash) *types.Transaction
+
// AddRemotes should add the given transactions to the pool.
AddRemotes([]*types.Transaction) []error
diff --git a/eth/protocol_test.go b/eth/protocol_test.go
index ca86664fce90..fec5b1762a68 100644
--- a/eth/protocol_test.go
+++ b/eth/protocol_test.go
@@ -20,6 +20,7 @@ import (
"fmt"
"math/big"
"sync"
+ "sync/atomic"
"testing"
"time"
@@ -61,7 +62,7 @@ func TestStatusMsgErrors63(t *testing.T) {
wantError error
}{
{
- code: TxMsg, data: []interface{}{},
+ code: TransactionMsg, data: []interface{}{},
wantError: errResp(ErrNoStatusMsg, "first msg has code 2 (!= 0)"),
},
{
@@ -113,7 +114,7 @@ func TestStatusMsgErrors64(t *testing.T) {
wantError error
}{
{
- code: TxMsg, data: []interface{}{},
+ code: TransactionMsg, data: []interface{}{},
wantError: errResp(ErrNoStatusMsg, "first msg has code 2 (!= 0)"),
},
{
@@ -180,16 +181,16 @@ func TestForkIDSplit(t *testing.T) {
blocksNoFork, _ = core.GenerateChain(configNoFork, genesisNoFork, engine, dbNoFork, 2, nil)
blocksProFork, _ = core.GenerateChain(configProFork, genesisProFork, engine, dbProFork, 2, nil)
- ethNoFork, _ = NewProtocolManager(configNoFork, downloader.FullSync, 1, new(event.TypeMux), new(testTxPool), engine, chainNoFork, dbNoFork)
- ethProFork, _ = NewProtocolManager(configProFork, downloader.FullSync, 1, new(event.TypeMux), new(testTxPool), engine, chainProFork, dbProFork)
+ ethNoFork, _ = NewProtocolManager(configNoFork, downloader.FullSync, 1, new(event.TypeMux), &testTxPool{pool: make(map[common.Hash]*types.Transaction)}, engine, chainNoFork, dbNoFork)
+ ethProFork, _ = NewProtocolManager(configProFork, downloader.FullSync, 1, new(event.TypeMux), &testTxPool{pool: make(map[common.Hash]*types.Transaction)}, engine, chainProFork, dbProFork)
)
ethNoFork.Start(1000)
ethProFork.Start(1000)
// Both nodes should allow the other to connect (same genesis, next fork is the same)
p2pNoFork, p2pProFork := p2p.MsgPipe()
- peerNoFork := newPeer(64, p2p.NewPeer(enode.ID{1}, "", nil), p2pNoFork)
- peerProFork := newPeer(64, p2p.NewPeer(enode.ID{2}, "", nil), p2pProFork)
+ peerNoFork := newPeer(64, p2p.NewPeer(enode.ID{1}, "", nil), p2pNoFork, nil)
+ peerProFork := newPeer(64, p2p.NewPeer(enode.ID{2}, "", nil), p2pProFork, nil)
errc := make(chan error, 2)
go func() { errc <- ethNoFork.handle(peerProFork) }()
@@ -207,8 +208,8 @@ func TestForkIDSplit(t *testing.T) {
chainProFork.InsertChain(blocksProFork[:1])
p2pNoFork, p2pProFork = p2p.MsgPipe()
- peerNoFork = newPeer(64, p2p.NewPeer(enode.ID{1}, "", nil), p2pNoFork)
- peerProFork = newPeer(64, p2p.NewPeer(enode.ID{2}, "", nil), p2pProFork)
+ peerNoFork = newPeer(64, p2p.NewPeer(enode.ID{1}, "", nil), p2pNoFork, nil)
+ peerProFork = newPeer(64, p2p.NewPeer(enode.ID{2}, "", nil), p2pProFork, nil)
errc = make(chan error, 2)
go func() { errc <- ethNoFork.handle(peerProFork) }()
@@ -226,8 +227,8 @@ func TestForkIDSplit(t *testing.T) {
chainProFork.InsertChain(blocksProFork[1:2])
p2pNoFork, p2pProFork = p2p.MsgPipe()
- peerNoFork = newPeer(64, p2p.NewPeer(enode.ID{1}, "", nil), p2pNoFork)
- peerProFork = newPeer(64, p2p.NewPeer(enode.ID{2}, "", nil), p2pProFork)
+ peerNoFork = newPeer(64, p2p.NewPeer(enode.ID{1}, "", nil), p2pNoFork, nil)
+ peerProFork = newPeer(64, p2p.NewPeer(enode.ID{2}, "", nil), p2pProFork, nil)
errc = make(chan error, 2)
go func() { errc <- ethNoFork.handle(peerProFork) }()
@@ -246,6 +247,7 @@ func TestForkIDSplit(t *testing.T) {
// This test checks that received transactions are added to the local pool.
func TestRecvTransactions63(t *testing.T) { testRecvTransactions(t, 63) }
func TestRecvTransactions64(t *testing.T) { testRecvTransactions(t, 64) }
+func TestRecvTransactions65(t *testing.T) { testRecvTransactions(t, 65) }
func testRecvTransactions(t *testing.T, protocol int) {
txAdded := make(chan []*types.Transaction)
@@ -256,7 +258,7 @@ func testRecvTransactions(t *testing.T, protocol int) {
defer p.close()
tx := newTestTransaction(testAccount, 0, 0)
- if err := p2p.Send(p.app, TxMsg, []interface{}{tx}); err != nil {
+ if err := p2p.Send(p.app, TransactionMsg, []interface{}{tx}); err != nil {
t.Fatalf("send error: %v", err)
}
select {
@@ -274,18 +276,22 @@ func testRecvTransactions(t *testing.T, protocol int) {
// This test checks that pending transactions are sent.
func TestSendTransactions63(t *testing.T) { testSendTransactions(t, 63) }
func TestSendTransactions64(t *testing.T) { testSendTransactions(t, 64) }
+func TestSendTransactions65(t *testing.T) { testSendTransactions(t, 65) }
func testSendTransactions(t *testing.T, protocol int) {
pm, _ := newTestProtocolManagerMust(t, downloader.FullSync, 0, nil, nil)
defer pm.Stop()
- // Fill the pool with big transactions.
+ // Fill the pool with big transactions (use a subscription to wait until all
+ // the transactions are announced to avoid spurious events causing extra
+ // broadcasts).
const txsize = txsyncPackSize / 10
alltxs := make([]*types.Transaction, 100)
for nonce := range alltxs {
alltxs[nonce] = newTestTransaction(testAccount, uint64(nonce), txsize)
}
pm.txpool.AddRemotes(alltxs)
+ time.Sleep(100 * time.Millisecond) // Wait until new tx even gets out of the system (lame)
// Connect several peers. They should all receive the pending transactions.
var wg sync.WaitGroup
@@ -297,18 +303,50 @@ func testSendTransactions(t *testing.T, protocol int) {
seen[tx.Hash()] = false
}
for n := 0; n < len(alltxs) && !t.Failed(); {
- var txs []*types.Transaction
- msg, err := p.app.ReadMsg()
- if err != nil {
- t.Errorf("%v: read error: %v", p.Peer, err)
- } else if msg.Code != TxMsg {
- t.Errorf("%v: got code %d, want TxMsg", p.Peer, msg.Code)
- }
- if err := msg.Decode(&txs); err != nil {
- t.Errorf("%v: %v", p.Peer, err)
+ var forAllHashes func(callback func(hash common.Hash))
+ switch protocol {
+ case 63:
+ fallthrough
+ case 64:
+ msg, err := p.app.ReadMsg()
+ if err != nil {
+ t.Errorf("%v: read error: %v", p.Peer, err)
+ continue
+ } else if msg.Code != TransactionMsg {
+ t.Errorf("%v: got code %d, want TxMsg", p.Peer, msg.Code)
+ continue
+ }
+ var txs []*types.Transaction
+ if err := msg.Decode(&txs); err != nil {
+ t.Errorf("%v: %v", p.Peer, err)
+ continue
+ }
+ forAllHashes = func(callback func(hash common.Hash)) {
+ for _, tx := range txs {
+ callback(tx.Hash())
+ }
+ }
+ case 65:
+ msg, err := p.app.ReadMsg()
+ if err != nil {
+ t.Errorf("%v: read error: %v", p.Peer, err)
+ continue
+ } else if msg.Code != NewPooledTransactionHashesMsg {
+ t.Errorf("%v: got code %d, want NewPooledTransactionHashesMsg", p.Peer, msg.Code)
+ continue
+ }
+ var hashes []common.Hash
+ if err := msg.Decode(&hashes); err != nil {
+ t.Errorf("%v: %v", p.Peer, err)
+ continue
+ }
+ forAllHashes = func(callback func(hash common.Hash)) {
+ for _, h := range hashes {
+ callback(h)
+ }
+ }
}
- for _, tx := range txs {
- hash := tx.Hash()
+ forAllHashes(func(hash common.Hash) {
seentx, want := seen[hash]
if seentx {
t.Errorf("%v: got tx more than once: %x", p.Peer, hash)
@@ -318,7 +356,7 @@ func testSendTransactions(t *testing.T, protocol int) {
}
seen[hash] = true
n++
- }
+ })
}
}
for i := 0; i < 3; i++ {
@@ -329,6 +367,53 @@ func testSendTransactions(t *testing.T, protocol int) {
wg.Wait()
}
+func TestTransactionPropagation(t *testing.T) { testSyncTransaction(t, true) }
+func TestTransactionAnnouncement(t *testing.T) { testSyncTransaction(t, false) }
+
+func testSyncTransaction(t *testing.T, propagtion bool) {
+ // Create a protocol manager for transaction fetcher and sender
+ pmFetcher, _ := newTestProtocolManagerMust(t, downloader.FastSync, 0, nil, nil)
+ defer pmFetcher.Stop()
+ pmSender, _ := newTestProtocolManagerMust(t, downloader.FastSync, 1024, nil, nil)
+ pmSender.broadcastTxAnnouncesOnly = !propagtion
+ defer pmSender.Stop()
+
+ // Sync up the two peers
+ io1, io2 := p2p.MsgPipe()
+
+ go pmSender.handle(pmSender.newPeer(65, p2p.NewPeer(enode.ID{}, "sender", nil), io2, pmSender.txpool.Get))
+ go pmFetcher.handle(pmFetcher.newPeer(65, p2p.NewPeer(enode.ID{}, "fetcher", nil), io1, pmFetcher.txpool.Get))
+
+ time.Sleep(250 * time.Millisecond)
+ pmFetcher.synchronise(pmFetcher.peers.BestPeer())
+ atomic.StoreUint32(&pmFetcher.acceptTxs, 1)
+
+ newTxs := make(chan core.NewTxsEvent, 1024)
+ sub := pmFetcher.txpool.SubscribeNewTxsEvent(newTxs)
+ defer sub.Unsubscribe()
+
+ // Fill the pool with new transactions
+ alltxs := make([]*types.Transaction, 1024)
+ for nonce := range alltxs {
+ alltxs[nonce] = newTestTransaction(testAccount, uint64(nonce), 0)
+ }
+ pmSender.txpool.AddRemotes(alltxs)
+
+ var got int
+loop:
+ for {
+ select {
+ case ev := <-newTxs:
+ got += len(ev.Txs)
+ if got == 1024 {
+ break loop
+ }
+ case <-time.NewTimer(time.Second).C:
+ t.Fatal("Failed to retrieve all transaction")
+ }
+ }
+}
+
// Tests that the custom union field encoder and decoder works correctly.
func TestGetBlockHeadersDataEncodeDecode(t *testing.T) {
// Create a "random" hash for testing
diff --git a/eth/sync.go b/eth/sync.go
index df306fba4bc5..73e3796e71f2 100644
--- a/eth/sync.go
+++ b/eth/sync.go
@@ -44,6 +44,12 @@ type txsync struct {
// syncTransactions starts sending all currently pending transactions to the given peer.
func (pm *ProtocolManager) syncTransactions(p *peer) {
+ // Assemble the set of transaction to broadcast or announce to the remote
+ // peer. Fun fact, this is quite an expensive operation as it needs to sort
+ // the transactions if the sorting is not cached yet. However, with a random
+ // order, insertions could overflow the non-executable queues and get dropped.
+ //
+ // TODO(karalabe): Figure out if we could get away with random order somehow
var txs types.Transactions
pending, _ := pm.txpool.Pending()
for _, batch := range pending {
@@ -52,26 +58,40 @@ func (pm *ProtocolManager) syncTransactions(p *peer) {
if len(txs) == 0 {
return
}
+ // The eth/65 protocol introduces proper transaction announcements, so instead
+ // of dripping transactions across multiple peers, just send the entire list as
+ // an announcement and let the remote side decide what they need (likely nothing).
+ if p.version >= eth65 {
+ hashes := make([]common.Hash, len(txs))
+ for i, tx := range txs {
+ hashes[i] = tx.Hash()
+ }
+ p.AsyncSendPooledTransactionHashes(hashes)
+ return
+ }
+ // Out of luck, peer is running legacy protocols, drop the txs over
select {
- case pm.txsyncCh <- &txsync{p, txs}:
+ case pm.txsyncCh <- &txsync{p: p, txs: txs}:
case <-pm.quitSync:
}
}
-// txsyncLoop takes care of the initial transaction sync for each new
+// txsyncLoop64 takes care of the initial transaction sync for each new
// connection. When a new peer appears, we relay all currently pending
// transactions. In order to minimise egress bandwidth usage, we send
// the transactions in small packs to one peer at a time.
-func (pm *ProtocolManager) txsyncLoop() {
+func (pm *ProtocolManager) txsyncLoop64() {
var (
pending = make(map[enode.ID]*txsync)
sending = false // whether a send is active
pack = new(txsync) // the pack that is being sent
done = make(chan error, 1) // result of the send
)
-
// send starts a sending a pack of transactions from the sync.
send := func(s *txsync) {
+ if s.p.version >= eth65 {
+ panic("initial transaction syncer running on eth/65+")
+ }
// Fill pack with transactions up to the target size.
size := common.StorageSize(0)
pack.p = s.p
@@ -88,7 +108,7 @@ func (pm *ProtocolManager) txsyncLoop() {
// Send the pack in the background.
s.p.Log().Trace("Sending batch of transactions", "count", len(pack.txs), "bytes", size)
sending = true
- go func() { done <- pack.p.SendTransactions(pack.txs) }()
+ go func() { done <- pack.p.SendTransactions64(pack.txs) }()
}
// pick chooses the next pending sync.
@@ -133,10 +153,10 @@ func (pm *ProtocolManager) txsyncLoop() {
// downloading hashes and blocks as well as handling the announcement handler.
func (pm *ProtocolManager) syncer() {
// Start and ensure cleanup of sync mechanisms
- pm.fetcher.Start()
- pm.bft.Start()
- defer pm.fetcher.Stop()
- defer pm.bft.Stop()
+ pm.blockFetcher.Start()
+ pm.txFetcher.Start()
+ defer pm.blockFetcher.Stop()
+ defer pm.txFetcher.Stop()
defer pm.downloader.Terminate()
// Wait for different events to fire synchronisation operations
diff --git a/eth/sync_test.go b/eth/sync_test.go
index 8bbb6a7ec323..5583543dff2b 100644
--- a/eth/sync_test.go
+++ b/eth/sync_test.go
@@ -26,9 +26,13 @@ import (
"github.com/XinFinOrg/XDPoSChain/p2p/enode"
)
+func TestFastSyncDisabling63(t *testing.T) { testFastSyncDisabling(t, 63) }
+func TestFastSyncDisabling64(t *testing.T) { testFastSyncDisabling(t, 64) }
+func TestFastSyncDisabling65(t *testing.T) { testFastSyncDisabling(t, 65) }
+
// Tests that fast sync gets disabled as soon as a real block is successfully
// imported into the blockchain.
-func TestFastSyncDisabling(t *testing.T) {
+func testFastSyncDisabling(t *testing.T, protocol int) {
// Create a pristine protocol manager, check that fast sync is left enabled
pmEmpty, _ := newTestProtocolManagerMust(t, downloader.FastSync, 0, nil, nil)
if atomic.LoadUint32(&pmEmpty.fastSync) == 0 {
@@ -42,8 +46,8 @@ func TestFastSyncDisabling(t *testing.T) {
// Sync up the two peers
io1, io2 := p2p.MsgPipe()
- go pmFull.handle(pmFull.newPeer(63, p2p.NewPeer(enode.ID{}, "empty", nil), io2))
- go pmEmpty.handle(pmEmpty.newPeer(63, p2p.NewPeer(enode.ID{}, "full", nil), io1))
+ go pmFull.handle(pmFull.newPeer(protocol, p2p.NewPeer(enode.ID{}, "empty", nil), io2, pmFull.txpool.Get))
+ go pmEmpty.handle(pmEmpty.newPeer(protocol, p2p.NewPeer(enode.ID{}, "full", nil), io1, pmEmpty.txpool.Get))
time.Sleep(250 * time.Millisecond)
pmEmpty.synchronise(pmEmpty.peers.BestPeer())
diff --git a/fuzzbuzz.yaml b/fuzzbuzz.yaml
new file mode 100644
index 000000000000..2a4f0c296fe7
--- /dev/null
+++ b/fuzzbuzz.yaml
@@ -0,0 +1,44 @@
+# bmt keystore rlp trie whisperv6
+
+base: ubuntu:16.04
+targets:
+ - name: rlp
+ language: go
+ version: "1.13"
+ corpus: ./fuzzers/rlp/corpus
+ harness:
+ function: Fuzz
+ package: github.com/ethereum/go-ethereum/tests/fuzzers/rlp
+ checkout: github.com/ethereum/go-ethereum/
+ - name: keystore
+ language: go
+ version: "1.13"
+ corpus: ./fuzzers/keystore/corpus
+ harness:
+ function: Fuzz
+ package: github.com/ethereum/go-ethereum/tests/fuzzers/keystore
+ checkout: github.com/ethereum/go-ethereum/
+ - name: trie
+ language: go
+ version: "1.13"
+ corpus: ./fuzzers/trie/corpus
+ harness:
+ function: Fuzz
+ package: github.com/ethereum/go-ethereum/tests/fuzzers/trie
+ checkout: github.com/ethereum/go-ethereum/
+ - name: txfetcher
+ language: go
+ version: "1.13"
+ corpus: ./fuzzers/txfetcher/corpus
+ harness:
+ function: Fuzz
+ package: github.com/ethereum/go-ethereum/tests/fuzzers/txfetcher
+ checkout: github.com/ethereum/go-ethereum/
+ - name: whisperv6
+ language: go
+ version: "1.13"
+ corpus: ./fuzzers/whisperv6/corpus
+ harness:
+ function: Fuzz
+ package: github.com/ethereum/go-ethereum/tests/fuzzers/whisperv6
+ checkout: github.com/ethereum/go-ethereum/
diff --git a/tests/fuzzers/txfetcher/corpus/0151ee1d0db4c74d3bcdfa4f7396a4c8538748c9-2 b/tests/fuzzers/txfetcher/corpus/0151ee1d0db4c74d3bcdfa4f7396a4c8538748c9-2
new file mode 100644
index 000000000000..2c75e9c7a755
--- /dev/null
+++ b/tests/fuzzers/txfetcher/corpus/0151ee1d0db4c74d3bcdfa4f7396a4c8538748c9-2
@@ -0,0 +1 @@
+¿½
\ No newline at end of file
diff --git a/tests/fuzzers/txfetcher/corpus/020dd7b492a6eb34ff0b7d8ee46189422c37e4a7-6 b/tests/fuzzers/txfetcher/corpus/020dd7b492a6eb34ff0b7d8ee46189422c37e4a7-6
new file mode 100644
index 000000000000..8d3b57789e79
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/020dd7b492a6eb34ff0b7d8ee46189422c37e4a7-6 differ
diff --git a/tests/fuzzers/txfetcher/corpus/021d1144e359233c496e22c3250609b11b213e9f-4 b/tests/fuzzers/txfetcher/corpus/021d1144e359233c496e22c3250609b11b213e9f-4
new file mode 100644
index 000000000000..73731899d588
--- /dev/null
+++ b/tests/fuzzers/txfetcher/corpus/021d1144e359233c496e22c3250609b11b213e9f-4
@@ -0,0 +1,12 @@
+ TESTING KEY-----
+MIICXgIBAAKBgQDuLnQAI3mDgey3VBzWnB2L39JUU4txjeVE6myuDqkM/uGlfjb9
+SjY1bIw4iAJm2gsvvZhIrCHS3l6afab4pZB
+l2+XsDlrKBxKKtDrGxlG4LjncdabFn9gvLZad2bSysqz/qTAUStTtqJQIDAQAB
+AoGAGRzwwir7XvBOAy5tuV6ef6anZzus1s1Y1Clb6HbnWWF/wbZGOpet
+3m4vD6MXc7jpTLryzTQIvVdfQbRc6+MUVeLKZTXtdZrh+k7hx0nTP8Jcb
+uqFk541awmMogY/EfbWd6IOkp+4xqjlFBEDytgbIECQQDvH/6nk+hgN4H
+qzzVtxxr397vWrjrIgPbJpQvBsafG7b0dA4AFjwVbFLmQcj2PprIMmPcQrooz84SHEg1Ak/7KCxmD/sfgS5TeuNi8DoUBEmiSJwm7FX
+ftxuvL7XvjwjN5B30pNEbc6Iuyt7y4MQJBAIt21su43sjXNueLKH8+ph2UfQuU9txblTu14q3N7gHRZB4ZMhFYyDy8CKrN2cPg/Fvyt0Xl/DoCzjA0CQQDU
+y2pGsuSmgUtWj3NM9xuwYPm+Z/F84K6+ARYiZ6PYj013sovGKUFfYAqVXVlxtIáo‡X
+qUn3Xh9ps8ZfjLZO7BAkEAlT4R5Yl6cGhaJQYZHOde3JMhNRcVFMO8dDaFo
+f9Oeos0UotgiDktdQHxdNEwLjQlJBz+OtwwA=---E RATTIEY-
\ No newline at end of file
diff --git a/tests/fuzzers/txfetcher/corpus/0d28327b1fb52c1ba02a6eb96675c31633921bb2-2 b/tests/fuzzers/txfetcher/corpus/0d28327b1fb52c1ba02a6eb96675c31633921bb2-2
new file mode 100644
index 000000000000..8cc3039cb837
--- /dev/null
+++ b/tests/fuzzers/txfetcher/corpus/0d28327b1fb52c1ba02a6eb96675c31633921bb2-2
@@ -0,0 +1,15 @@
+¸&^£áo‡È—-----BEGIN RSA TESTING KEY-----
+MIICXgIBAAKBgQDuLnQAI3mDgey3VBzWnB2L39JUU4txjeVE6myuDqkM/uGlfjb9
+SjY1bIw4iA5sBBZzHi3z0h1YV8QPuxEbi4nW91IJm2gsvvZhIrCHS3l6afab4pZB
+l2+XsDulrKBxKKtD1rGxlG4LjncdabFn9gvLZad2bSysqz/qTAUStTvqJQIDAQAB
+AoGAGRzwwir7XvBOAy5tM/uV6e+Zf6anZzus1s1Y1ClbjbE6HXbnWWF/wbZGOpet
+3Zm4vD6MXc7jpTLryzTQIvVdfQbRc6+MUVeLKwZatTXtdZrhu+Jk7hx0nTPy8Jcb
+uJqFk541aEw+mMogY/xEcfbWd6IOkp+4xqjlFLBEDytgbIECQQDvH/E6nk+hgN4H
+qzzVtxxr397vWrjrIgPbJpQvBsafG7b0dA4AFjwVbFLmQcj2PprIMmPcQrooz8vp
+jy4SHEg1AkEA/v13/5M47K9vCxmb8QeD/asydfsgS5TeuNi8DoUBEmiSJwma7FXY
+fFUtxuvL7XvjwjN5B30pNEbc6Iuyt7y4MQJBAIt21su4b3sjXNueLKH85Q+phy2U
+fQtuUE9txblTu14q3N7gHRZB4ZMhFYyDy8CKrN2cPg/Fvyt0Xlp/DoCzjA0CQQDU
+y2ptGsuSmgUtWj3NM9xuwYPm+Z/F84K6+ARYiZ6PYj013sovGKUFfYAqVXVlxtIX
+qyUBnu3X9ps8ZfjLZO7BAkEAlT4R5Yl6cGhaJQYZHOde3JEMhNRcVFMO8dJDaFeo
+f9Oeos0UUothgiDktdQHxdNEwLjQf7lJJBzV+5OtwswCWA==
+-----END RSA TESTING KEY-----Q_
\ No newline at end of file
diff --git a/tests/fuzzers/txfetcher/corpus/0fcd827b57ded58e91f7ba2ac2b7ea4d25ebedca-7 b/tests/fuzzers/txfetcher/corpus/0fcd827b57ded58e91f7ba2ac2b7ea4d25ebedca-7
new file mode 100644
index 000000000000..8ceee16af1ee
--- /dev/null
+++ b/tests/fuzzers/txfetcher/corpus/0fcd827b57ded58e91f7ba2ac2b7ea4d25ebedca-7
@@ -0,0 +1 @@
+ð½apï¿ïï��ï¿ï¿¿½½½¿¿½½ï¿½ï¿½¿½ï¿ï¿½ï¿ïÓÌV½¿½ï ïï¿ï¿½#ï¿ï¿½&��
\ No newline at end of file
diff --git a/tests/fuzzers/txfetcher/corpus/109bc9b8fd4fef63493e104c703c79bc4a5e8d34-6 b/tests/fuzzers/txfetcher/corpus/109bc9b8fd4fef63493e104c703c79bc4a5e8d34-6
new file mode 100644
index 000000000000..df9b986af100
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/109bc9b8fd4fef63493e104c703c79bc4a5e8d34-6 differ
diff --git a/tests/fuzzers/txfetcher/corpus/163785ab002746452619f31e8dfcb4549e6f8b6e-6 b/tests/fuzzers/txfetcher/corpus/163785ab002746452619f31e8dfcb4549e6f8b6e-6
new file mode 100644
index 000000000000..55467373d461
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/163785ab002746452619f31e8dfcb4549e6f8b6e-6 differ
diff --git a/tests/fuzzers/txfetcher/corpus/1adfa6b9ddf5766220c8ff7ede2926ca241bb947-3 b/tests/fuzzers/txfetcher/corpus/1adfa6b9ddf5766220c8ff7ede2926ca241bb947-3
new file mode 100644
index 000000000000..4a593aa28dd4
--- /dev/null
+++ b/tests/fuzzers/txfetcher/corpus/1adfa6b9ddf5766220c8ff7ede2926ca241bb947-3
@@ -0,0 +1,11 @@
+TAKBgDuLnQA3gey3VBznB39JUtxjeE6myuDkM/uGlfjb
+S1w4iA5sBzzh8uxEbi4nW91IJm2gsvvZhICHS3l6ab4pZB
+l2DulrKBxKKtD1rGxlG4LncabFn9vLZad2bSysqz/qTAUSTvqJQIDAQAB
+AoGAGRzwwir7XvBOAy5tM/uV6e+Zf6anZzus1s1Y1ClbjbE6HXbnWWF/wbZGOpet
+3Z4vMXc7jpTLryzTQIvVdfQbRc6+MUVeLKZatTXtdZrhu+Jk7hx0nTPy8Jcb
+uJqFk54MogxEcfbWd6IOkp+4xqFLBEDtgbIECnk+hgN4H
+qzzxxr397vWrjrIgbJpQvBv8QeeuNi8DoUBEmiSJwa7FXY
+FUtxuvL7XvjwjN5B30pEbc6Iuyt7y4MQJBAIt21su4b3sjphy2tuUE9xblTu14qgHZ6+AiZovGKU--FfYAqVXVlxtIX
+qyU3X9ps8ZfjLZ45l6cGhaJQYZHOde3JEMhNRcVFMO8dJDaFeo
+f9Oeos0UUothgiDktdQHxdNEwLjQf7lJJBzV+5OtwswCWA==
+-----END RSA T
\ No newline at end of file
diff --git a/tests/fuzzers/txfetcher/corpus/1b9a02e9a48fea1d2fc3fb77946ada278e152079-4 b/tests/fuzzers/txfetcher/corpus/1b9a02e9a48fea1d2fc3fb77946ada278e152079-4
new file mode 100644
index 000000000000..4a56f93d3ba9
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/1b9a02e9a48fea1d2fc3fb77946ada278e152079-4 differ
diff --git a/tests/fuzzers/txfetcher/corpus/1e14c7ea1faef92890988061b5abe96db7190f98-7 b/tests/fuzzers/txfetcher/corpus/1e14c7ea1faef92890988061b5abe96db7190f98-7
new file mode 100644
index 000000000000..d2442fc5a6c5
--- /dev/null
+++ b/tests/fuzzers/txfetcher/corpus/1e14c7ea1faef92890988061b5abe96db7190f98-7
@@ -0,0 +1 @@
+0000000000000000000000000000000000000000000000000000000000000000000000000
\ No newline at end of file
diff --git a/tests/fuzzers/txfetcher/corpus/1e7d05f00e99cbf3ff0ef1cd7ea8dd07ad6dff23-6 b/tests/fuzzers/txfetcher/corpus/1e7d05f00e99cbf3ff0ef1cd7ea8dd07ad6dff23-6
new file mode 100644
index 000000000000..1c342ff53a36
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/1e7d05f00e99cbf3ff0ef1cd7ea8dd07ad6dff23-6 differ
diff --git a/tests/fuzzers/txfetcher/corpus/1ec95e347fd522e6385b5091aa81aa2485be4891-4 b/tests/fuzzers/txfetcher/corpus/1ec95e347fd522e6385b5091aa81aa2485be4891-4
new file mode 100644
index 000000000000..b0c776bd4d99
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/1ec95e347fd522e6385b5091aa81aa2485be4891-4 differ
diff --git a/tests/fuzzers/txfetcher/corpus/1fbfa5d214060d2a0905846a589fd6f78d411451-4 b/tests/fuzzers/txfetcher/corpus/1fbfa5d214060d2a0905846a589fd6f78d411451-4
new file mode 100644
index 000000000000..75de835c98de
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/1fbfa5d214060d2a0905846a589fd6f78d411451-4 differ
diff --git a/tests/fuzzers/txfetcher/corpus/1fd84ee194e791783a7f18f0a6deab8efe05fc04-2 b/tests/fuzzers/txfetcher/corpus/1fd84ee194e791783a7f18f0a6deab8efe05fc04-2
new file mode 100644
index 000000000000..3b6d2560aea8
--- /dev/null
+++ b/tests/fuzzers/txfetcher/corpus/1fd84ee194e791783a7f18f0a6deab8efe05fc04-2
@@ -0,0 +1 @@
+¸&
\ No newline at end of file
diff --git a/tests/fuzzers/txfetcher/corpus/21e76b9fca21d94d97f860c1c82f40697a83471b-8 b/tests/fuzzers/txfetcher/corpus/21e76b9fca21d94d97f860c1c82f40697a83471b-8
new file mode 100644
index 000000000000..1d4620f49f21
--- /dev/null
+++ b/tests/fuzzers/txfetcher/corpus/21e76b9fca21d94d97f860c1c82f40697a83471b-8
@@ -0,0 +1,3 @@
+DtQvfQ+MULKZTXk78c
+/fWkpxlQQ/+hgNzVtx9vWgJsafG7b0dA4AFjwVbFLmQcj2PprIMmPNQrooX
+L
\ No newline at end of file
diff --git a/tests/fuzzers/txfetcher/corpus/220a87fed0c92474923054094eb7aff14289cf5e-4 b/tests/fuzzers/txfetcher/corpus/220a87fed0c92474923054094eb7aff14289cf5e-4
new file mode 100644
index 000000000000..175f74fd5aa8
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/220a87fed0c92474923054094eb7aff14289cf5e-4 differ
diff --git a/tests/fuzzers/txfetcher/corpus/23ddcd66aa92fe3d78b7f5b6e7cddb1b55c5f5df-3 b/tests/fuzzers/txfetcher/corpus/23ddcd66aa92fe3d78b7f5b6e7cddb1b55c5f5df-3
new file mode 100644
index 000000000000..95892c7b00c5
--- /dev/null
+++ b/tests/fuzzers/txfetcher/corpus/23ddcd66aa92fe3d78b7f5b6e7cddb1b55c5f5df-3
@@ -0,0 +1,12 @@
+4txjeVE6myuDqkM/uGlfjb9
+SjY1bIw4iA5sBBZzHi3z0h1YV8QPuxEbi4nW91IJm2gsvvZeIrCHS3l6afab4pZB
+l2+XsDlrKBxKKtD1rGxlG4jncdabFn9gvLZad2bSysqz/qTAUSTvqJQIDAQAB
+AoGAGRzwwXvBOAy5tM/uV6e+Zf6aZzus1s1Y1ClbjbE6HXbnWWF/wbZGOpet
+3Z4vD6Mc7pLryzTQIVdfQbRc6+MUVeLKZaTXtdZru+Jk70PJJqFk541aEw+mMogY/xEcfbWd6IOkp+4xqjlFLBEDytgbIECQQDvH/E6nk+gN4H
+qzzVtxxr397vWrjrIgPbJpQvBsafG7b0dA4AFjwVbFLmQ2PprIMPcQroo8vpjSHg1Ev14KxmQeDydfsgeuN8UBESJwm7F
+UtuL7Xvjw50pNEbc6Iuyty4QJA21su4sjXNueLQphy2U
+fQtuUE9txblTu14qN7gHRZB4ZMhFYyDy8CKrN2cPg/Fvyt0Xlp/DoCzjA0CQQDU
+y2ptGsuSmgUtWj3NM9xuwYPm+Z/F84K6ARYiZPYj1oGUFfYAVVxtI
+qyBnu3X9pfLZOAkEAlT4R5Yl6cJQYZHOde3JEhNRcVFMO8dJFo
+f9Oeos0UUhgiDkQxdEwLjQf7lJJz5OtwC=
+-NRSA TESINGKEY-Q_
\ No newline at end of file
diff --git a/tests/fuzzers/txfetcher/corpus/2441d249faf9a859e38c49f6e305b394280c6ea5-1 b/tests/fuzzers/txfetcher/corpus/2441d249faf9a859e38c49f6e305b394280c6ea5-1
new file mode 100644
index 000000000000..d76207e992a6
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/2441d249faf9a859e38c49f6e305b394280c6ea5-1 differ
diff --git a/tests/fuzzers/txfetcher/corpus/2da1f0635e11283b1927974f418aadd8837ad31e-7 b/tests/fuzzers/txfetcher/corpus/2da1f0635e11283b1927974f418aadd8837ad31e-7
new file mode 100644
index 000000000000..73ae7057014f
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/2da1f0635e11283b1927974f418aadd8837ad31e-7 differ
diff --git a/tests/fuzzers/txfetcher/corpus/2e1853fbf8efe40098b1583224fe3b5f335e7037-6 b/tests/fuzzers/txfetcher/corpus/2e1853fbf8efe40098b1583224fe3b5f335e7037-6
new file mode 100644
index 000000000000..692981e61415
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/2e1853fbf8efe40098b1583224fe3b5f335e7037-6 differ
diff --git a/tests/fuzzers/txfetcher/corpus/2f25490dc49c103d653843ed47324b310ee7105e-7 b/tests/fuzzers/txfetcher/corpus/2f25490dc49c103d653843ed47324b310ee7105e-7
new file mode 100644
index 000000000000..5cf7da75df2d
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/2f25490dc49c103d653843ed47324b310ee7105e-7 differ
diff --git a/tests/fuzzers/txfetcher/corpus/30494b85bb60ad7f099fa49d427007a761620d8f-5 b/tests/fuzzers/txfetcher/corpus/30494b85bb60ad7f099fa49d427007a761620d8f-5
new file mode 100644
index 000000000000..7ff9d397521d
--- /dev/null
+++ b/tests/fuzzers/txfetcher/corpus/30494b85bb60ad7f099fa49d427007a761620d8f-5
@@ -0,0 +1,10 @@
+jXbnWWF/wbZGOpet
+3Zm4vD6MXc7jpTLryzTQIvVdfQbRc6+MUVeLKwZatTXtdZrhu+Jk7hx0nTPy8Jcb
+uJqFk541aEw+mMogY/xEcfbWd6IOkp+4xqjlFLBEDytgbIECQQDvH/E6nk+hgN4H
+qzzVtxxr397vWrjrIgPbJpQvBsafG7b0dA4AFjwVbFLmQcj2PprIMmPcQrooz8vp
+jy4SHEg1AkEA/v13/5M47K9vCxb8QeD/asydfsgS5TeuNi8DoUBEmiSJwma7FXY
+fFUtxuvL7XvjwjN5B30pNEbc6Iuyt7y4MQJBAIt21su4b3sjXNueLKH85Q+phy2U
+fQtuUE9txblTu14q3N7gHRZB4ZMhFYyDy8CKrN2cPg/Fvyt0Xl/DoCzjA0CQQDU
+y2ptGsuSmgUtWj3NM9xuwYPm+Z/F84K6+ARYiZ6Yj013sovGKUFfYAqVXVlxtIX
+qyUBnu3Xh9ps8ZfjLZO7BAkEAlT4R5Yl6cGhaJQYZHOde3JEMhNRcVFMO8dDaFeo
+f9Oeos0UotgiDktdQHxdNEwLjQfl
\ No newline at end of file
diff --git a/tests/fuzzers/txfetcher/corpus/316024ca3aaf09c1de5258733ff5fe3d799648d3-4 b/tests/fuzzers/txfetcher/corpus/316024ca3aaf09c1de5258733ff5fe3d799648d3-4
new file mode 100644
index 000000000000..61f7d78f3463
--- /dev/null
+++ b/tests/fuzzers/txfetcher/corpus/316024ca3aaf09c1de5258733ff5fe3d799648d3-4
@@ -0,0 +1,15 @@
+¸^áo‡È—----BEGIN RA TTING KEY-----
+IIXgIBAAKBQDuLnQI3mDgey3VBzWnB2L39JUU4txjeVE6myuDqkM/uGlfjb9
+SjY1bIw4iA5sBBZzHi3z0h1YV8QPuxEbi4nW91IJmgsvvZhrCHSl6afab4pZB
+l2+XsDulrKBxKKtD1rGxlG4LjcdabF9gvLZad2bSysqz/qTAUStTvqJQDAQAB
+AoGAGRzwwir7XvBOAy5tM/uV6e+Zf6anZzus1s1Y1ClbjbE6HXbnWWF/wbZGOpet
+3Z4vD6MXc7jpTLryzTQIvVdfQbRc6+MUVeLKwZatTXtdZrhu+Jk7hx0nTPy8Jcb
+uJqFk541aEw+mMogY/xEcfbWd6IOkp+4xqjlFLBEDytgbIECQQDvH/E6nk+hgN4H
+qzzVtxxr397vWrjrIgPbJpQvBsafG7b0dA4AFjwVbFLmQcj2PprIMmPcQrooz8vp
+jy4SHEg1AkEA/v13/5M47K9vCxmb8QeD/asydfsgS5TeuNi8DoUBEmiSJwma7FXY
+fFUtxuvL7XvjwjN5B30pNEbc6Iuyt7y4MQJBAIt21su4b3sjXNueLKH85Q+phy2U
+fQtuUE9txblTu14q3N7gHRZB4ZMhFYyDy8CKrN2cPg/Fvyt0Xlp/DoCzjA0CQQDU
+y2ptGsuSmgUtWj3NM9xuwYPm+Z/F84K6+ARYiZ6PYj043sovGKUFfYAqVXVlxtIX
+qyUBnu3X9ps8ZfjLZO7BAkEAlT4R5Yl6cGhaJQYZHOde3JEMhNRcVFMO8dJDaFeo
+f9Oeos0UUothgiDktdQHxdNEwLjQf7lJJBzV+5OtwswCWA==
+-----END RSA TESTING KEY-----Q_
\ No newline at end of file
diff --git a/tests/fuzzers/txfetcher/corpus/32a089e2c439a91f4c1b67a13d52429bcded0dd9-7 b/tests/fuzzers/txfetcher/corpus/32a089e2c439a91f4c1b67a13d52429bcded0dd9-7
new file mode 100644
index 000000000000..a986a9d8e753
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/32a089e2c439a91f4c1b67a13d52429bcded0dd9-7 differ
diff --git a/tests/fuzzers/txfetcher/corpus/33ec1dc0bfeb93d16edee3c07125fec6ac1aa17d-2 b/tests/fuzzers/txfetcher/corpus/33ec1dc0bfeb93d16edee3c07125fec6ac1aa17d-2
new file mode 100644
index 000000000000..d41771b86ce9
--- /dev/null
+++ b/tests/fuzzers/txfetcher/corpus/33ec1dc0bfeb93d16edee3c07125fec6ac1aa17d-2
@@ -0,0 +1 @@
+ï¿
\ No newline at end of file
diff --git a/tests/fuzzers/txfetcher/corpus/37a0d207700b52caa005ec8aeb344dcb13150ed2-5 b/tests/fuzzers/txfetcher/corpus/37a0d207700b52caa005ec8aeb344dcb13150ed2-5
new file mode 100644
index 000000000000..2f09c6e28f03
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/37a0d207700b52caa005ec8aeb344dcb13150ed2-5 differ
diff --git a/tests/fuzzers/txfetcher/corpus/382f59c66d0ddb6747d3177263279789ca15c2db-5 b/tests/fuzzers/txfetcher/corpus/382f59c66d0ddb6747d3177263279789ca15c2db-5
new file mode 100644
index 000000000000..84441ac37462
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/382f59c66d0ddb6747d3177263279789ca15c2db-5 differ
diff --git a/tests/fuzzers/txfetcher/corpus/3a010483a4ad8d7215447ce27e0fac3791235c99-4 b/tests/fuzzers/txfetcher/corpus/3a010483a4ad8d7215447ce27e0fac3791235c99-4
new file mode 100644
index 000000000000..28f5d99b986a
--- /dev/null
+++ b/tests/fuzzers/txfetcher/corpus/3a010483a4ad8d7215447ce27e0fac3791235c99-4
@@ -0,0 +1,7 @@
+
+lGAGRzwwir7XvBOAy5tM/uV6e+Zf6anZzus1s1Y1ClbjbE6HXbnWWF/wbZGOpet
+3Zm4vD6MXc7jpTLryzTQIvVdfQbRc6+MUVeLKwZatTXtdZrhu+Jk7hx0nTPy8Jcb
+uJqFk541aEw+mMogY/xEcfbWd6IOkp+4xqjlFLBEDytgbIECQQDvH/E6nk+hgN4H
+qzzVtxxr397vWrjrIgPbJpQvBsafG7b0dA4AFjwVbFLmQcj2PprIMmPcQrooz8vp
+jy4SHEg1AkEA/v13/5M47K9vCxmb8QeD/asydfsgS5TeuNi8DoUBEmiSJwma7FXY
+fFUtxuvL7XvjwjN5
\ No newline at end of file
diff --git a/tests/fuzzers/txfetcher/corpus/3a3b717fcfe7ffb000b906e5a76f32248a576bf7-6 b/tests/fuzzers/txfetcher/corpus/3a3b717fcfe7ffb000b906e5a76f32248a576bf7-6
new file mode 100644
index 000000000000..022de3c61d4b
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/3a3b717fcfe7ffb000b906e5a76f32248a576bf7-6 differ
diff --git a/tests/fuzzers/txfetcher/corpus/3c37f6d58b8029971935f127f53e6aaeba558445-6 b/tests/fuzzers/txfetcher/corpus/3c37f6d58b8029971935f127f53e6aaeba558445-6
new file mode 100644
index 000000000000..9f3bf093ad1c
--- /dev/null
+++ b/tests/fuzzers/txfetcher/corpus/3c37f6d58b8029971935f127f53e6aaeba558445-6
@@ -0,0 +1,2 @@
+¶Èíw¿½ï¿½Â€ï¿½ï¿½ï¿½ï¿½ï¿½ï¿½ï¿½ï¿½ï¿½ �
+� � �
���ï¿ï¿½ï¿½ï¿½ï¿½ï¿½ï¿½ï¿½ï¿½ï¿½ï¿½ï¿½ � � � � �!�"�#�$�%�&�'�(�)�*�+�,�-�.�/¿½0
\ No newline at end of file
diff --git a/tests/fuzzers/txfetcher/corpus/3c73b63bafa9f535c882ec17189adaf02b58f432-6 b/tests/fuzzers/txfetcher/corpus/3c73b63bafa9f535c882ec17189adaf02b58f432-6
new file mode 100644
index 000000000000..0dfbc46993f8
--- /dev/null
+++ b/tests/fuzzers/txfetcher/corpus/3c73b63bafa9f535c882ec17189adaf02b58f432-6
@@ -0,0 +1 @@
+LvhaJQHOe3EhRcdaFofeoogkjQfJB
\ No newline at end of file
diff --git a/tests/fuzzers/txfetcher/corpus/3d11500c4f66b20c73bbdfb1a7bddd7bbf92b29c-5 b/tests/fuzzers/txfetcher/corpus/3d11500c4f66b20c73bbdfb1a7bddd7bbf92b29c-5
new file mode 100644
index 000000000000..b19fc7f4584a
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/3d11500c4f66b20c73bbdfb1a7bddd7bbf92b29c-5 differ
diff --git a/tests/fuzzers/txfetcher/corpus/3d8b5bf36c80d6f65802280039f85421f32b5055-6 b/tests/fuzzers/txfetcher/corpus/3d8b5bf36c80d6f65802280039f85421f32b5055-6
new file mode 100644
index 000000000000..eacd269f317b
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/3d8b5bf36c80d6f65802280039f85421f32b5055-6 differ
diff --git a/tests/fuzzers/txfetcher/corpus/3f99c546a3962256176d566c19e3fffb62072078-1 b/tests/fuzzers/txfetcher/corpus/3f99c546a3962256176d566c19e3fffb62072078-1
new file mode 100644
index 000000000000..9e90183d6b65
--- /dev/null
+++ b/tests/fuzzers/txfetcher/corpus/3f99c546a3962256176d566c19e3fffb62072078-1
@@ -0,0 +1 @@
+¸&^£áo‡
\ No newline at end of file
diff --git a/tests/fuzzers/txfetcher/corpus/408ec46539af27acd82b3d01e863597030882458-8 b/tests/fuzzers/txfetcher/corpus/408ec46539af27acd82b3d01e863597030882458-8
new file mode 100644
index 000000000000..65d55437e5c5
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/408ec46539af27acd82b3d01e863597030882458-8 differ
diff --git a/tests/fuzzers/txfetcher/corpus/436154e5bb6487673f6642e6d2a582c01b083c08-8 b/tests/fuzzers/txfetcher/corpus/436154e5bb6487673f6642e6d2a582c01b083c08-8
new file mode 100644
index 000000000000..28e519c12589
--- /dev/null
+++ b/tests/fuzzers/txfetcher/corpus/436154e5bb6487673f6642e6d2a582c01b083c08-8
@@ -0,0 +1 @@
+ð½apfffffffffffffffffffffffffffffffebadce6f48a0Ÿ_3bbfd2364
\ No newline at end of file
diff --git a/tests/fuzzers/txfetcher/corpus/45f565cd14b8de1ba2e925047ce776c2682b4b8d-3 b/tests/fuzzers/txfetcher/corpus/45f565cd14b8de1ba2e925047ce776c2682b4b8d-3
new file mode 100644
index 000000000000..9f03a095b9f2
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/45f565cd14b8de1ba2e925047ce776c2682b4b8d-3 differ
diff --git a/tests/fuzzers/txfetcher/corpus/4a0a12f5b033c8c160cc3b5133692ea1e92c6cdf-7 b/tests/fuzzers/txfetcher/corpus/4a0a12f5b033c8c160cc3b5133692ea1e92c6cdf-7
new file mode 100644
index 000000000000..e50b5494c971
--- /dev/null
+++ b/tests/fuzzers/txfetcher/corpus/4a0a12f5b033c8c160cc3b5133692ea1e92c6cdf-7
@@ -0,0 +1,3 @@
+DtQvfQ+MULKZTXk78c
+/fWkpxlyEQQ/+hgNzVtx9vWgJsafG7b0dA4AFjwVbFLmQcj2PprIMmPNQg1Ak/7KCxmDgS5TDEmSJwFX
+txLjbt4xTgeXVlXsjLZ
\ No newline at end of file
diff --git a/tests/fuzzers/txfetcher/corpus/550f15ef65230cc4dcfab7fea67de212d9212ff8-8 b/tests/fuzzers/txfetcher/corpus/550f15ef65230cc4dcfab7fea67de212d9212ff8-8
new file mode 100644
index 000000000000..34005f43cbee
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/550f15ef65230cc4dcfab7fea67de212d9212ff8-8 differ
diff --git a/tests/fuzzers/txfetcher/corpus/5552213d659fef900a194c52718ffeffdc72d043-3 b/tests/fuzzers/txfetcher/corpus/5552213d659fef900a194c52718ffeffdc72d043-3
new file mode 100644
index 000000000000..7346ff1955e9
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/5552213d659fef900a194c52718ffeffdc72d043-3 differ
diff --git a/tests/fuzzers/txfetcher/corpus/5570ef82893a9b9b9158572d43a7de7537121d2d-1 b/tests/fuzzers/txfetcher/corpus/5570ef82893a9b9b9158572d43a7de7537121d2d-1
new file mode 100644
index 000000000000..feffcebca0c8
--- /dev/null
+++ b/tests/fuzzers/txfetcher/corpus/5570ef82893a9b9b9158572d43a7de7537121d2d-1
@@ -0,0 +1 @@
+ð½ï½ï¿½Ù¯0,1,2,3,4,5,6,7,-3420794409,(2,a)
\ No newline at end of file
diff --git a/tests/fuzzers/txfetcher/corpus/5e10f734f8af4116fbd164d96eec67aa53e6228c-5 b/tests/fuzzers/txfetcher/corpus/5e10f734f8af4116fbd164d96eec67aa53e6228c-5
new file mode 100644
index 000000000000..0eacd0b59a6d
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/5e10f734f8af4116fbd164d96eec67aa53e6228c-5 differ
diff --git a/tests/fuzzers/txfetcher/corpus/608200b402488b3989ec8ec5f4190ccb537b8ea4-4 b/tests/fuzzers/txfetcher/corpus/608200b402488b3989ec8ec5f4190ccb537b8ea4-4
new file mode 100644
index 000000000000..d37b018515b8
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/608200b402488b3989ec8ec5f4190ccb537b8ea4-4 differ
diff --git a/tests/fuzzers/txfetcher/corpus/61e89c3fbdf9eff74bd250ea73cc2e61f8ca0d97-5 b/tests/fuzzers/txfetcher/corpus/61e89c3fbdf9eff74bd250ea73cc2e61f8ca0d97-5
new file mode 100644
index 000000000000..155744bccc2f
--- /dev/null
+++ b/tests/fuzzers/txfetcher/corpus/61e89c3fbdf9eff74bd250ea73cc2e61f8ca0d97-5
@@ -0,0 +1 @@
+88242871'392752200424491531672177074144720616417147514758635765020556616¿
\ No newline at end of file
diff --git a/tests/fuzzers/txfetcher/corpus/62817a48c78fbf2c12fcdc5ca58e2ca60c43543a-7 b/tests/fuzzers/txfetcher/corpus/62817a48c78fbf2c12fcdc5ca58e2ca60c43543a-7
new file mode 100644
index 000000000000..795608a78957
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/62817a48c78fbf2c12fcdc5ca58e2ca60c43543a-7 differ
diff --git a/tests/fuzzers/txfetcher/corpus/6782da8f1a432a77306d60d2ac2470c35b98004f-3 b/tests/fuzzers/txfetcher/corpus/6782da8f1a432a77306d60d2ac2470c35b98004f-3
new file mode 100644
index 000000000000..f44949e6aefc
--- /dev/null
+++ b/tests/fuzzers/txfetcher/corpus/6782da8f1a432a77306d60d2ac2470c35b98004f-3
@@ -0,0 +1 @@
+21888242871'392752200424452601091531672177074144720616417147514758635765020556616¿½
\ No newline at end of file
diff --git a/tests/fuzzers/txfetcher/corpus/68fb55290cb9d6da5b259017c34bcecf96c944aa-5 b/tests/fuzzers/txfetcher/corpus/68fb55290cb9d6da5b259017c34bcecf96c944aa-5
new file mode 100644
index 000000000000..23d905b827e2
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/68fb55290cb9d6da5b259017c34bcecf96c944aa-5 differ
diff --git a/tests/fuzzers/txfetcher/corpus/6a5059bc86872526241d21ab5dae9f0afd3b9ae1-3 b/tests/fuzzers/txfetcher/corpus/6a5059bc86872526241d21ab5dae9f0afd3b9ae1-3
new file mode 100644
index 000000000000..b71d5dff5167
--- /dev/null
+++ b/tests/fuzzers/txfetcher/corpus/6a5059bc86872526241d21ab5dae9f0afd3b9ae1-3
@@ -0,0 +1 @@
+¿½
\ No newline at end of file
diff --git a/tests/fuzzers/txfetcher/corpus/717928e0e2d478c680c6409b173552ca98469ba5-6 b/tests/fuzzers/txfetcher/corpus/717928e0e2d478c680c6409b173552ca98469ba5-6
new file mode 100644
index 000000000000..dce51061150d
--- /dev/null
+++ b/tests/fuzzers/txfetcher/corpus/717928e0e2d478c680c6409b173552ca98469ba5-6
@@ -0,0 +1 @@
+LvhaJcdaFofenogkjQfJB
\ No newline at end of file
diff --git a/tests/fuzzers/txfetcher/corpus/71d22f25419543e437f249ca437823b87ac926b1-6 b/tests/fuzzers/txfetcher/corpus/71d22f25419543e437f249ca437823b87ac926b1-6
new file mode 100644
index 000000000000..d07a6c2f3244
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/71d22f25419543e437f249ca437823b87ac926b1-6 differ
diff --git a/tests/fuzzers/txfetcher/corpus/7312a0f31ae5d773ed4fd74abc7521eb14754683-8 b/tests/fuzzers/txfetcher/corpus/7312a0f31ae5d773ed4fd74abc7521eb14754683-8
new file mode 100644
index 000000000000..3593ce2e1931
--- /dev/null
+++ b/tests/fuzzers/txfetcher/corpus/7312a0f31ae5d773ed4fd74abc7521eb14754683-8
@@ -0,0 +1,2 @@
+DtQvfQ+MULKZTXk78c
+/fWkpxlyEQQ/+hgNzVtx9vWgJsafG7b0dA4AFjwVbFLmQcj2PprIMmPNQg1AkS5TDEmSJwFVlXsjLZ
\ No newline at end of file
diff --git a/tests/fuzzers/txfetcher/corpus/76e413a50dc8861e3756e556f796f1737bec2675-4 b/tests/fuzzers/txfetcher/corpus/76e413a50dc8861e3756e556f796f1737bec2675-4
new file mode 100644
index 000000000000..623fcf9601e5
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/76e413a50dc8861e3756e556f796f1737bec2675-4 differ
diff --git a/tests/fuzzers/txfetcher/corpus/78480977d5c07386b06e9b37f5c82f5ed86c2f09-3 b/tests/fuzzers/txfetcher/corpus/78480977d5c07386b06e9b37f5c82f5ed86c2f09-3
new file mode 100644
index 000000000000..e92863a1c703
--- /dev/null
+++ b/tests/fuzzers/txfetcher/corpus/78480977d5c07386b06e9b37f5c82f5ed86c2f09-3
@@ -0,0 +1,14 @@
+ TESTING KEY-----
+MIICXgIBAAKBgQDuLnQAI3mDgey3VBzWnB2L39JUU4txjeVE6myuDqkM/uGlfjb9
+SjY1bIw4iAJm2gsvvZhIrCHS3l6afab4pZB
+l2+XsDulrKBxKKtD1rGxlG4LjncdabFn9gvLZad2bSysqz/qTAUStTvqJQIDAQAB
+AoGAGRzwwir7XvBOAy5tM/uV6e+Zf6anZzus1s1Y1ClbjbE6HXbnWWF/wbZGOpet
+3Zm4vD6MXc7jpTLryzTQIvVdfQbRc6+MUVeLKwZatTXtdZrhu+Jk7hx0nTPy8Jcb
+uJqFk541aEw+mMogY/xEcfbWd6IOkp+4xqjlFLBEDytgbIECQQDvH/E6nk+hgN4H
+qzzVtxxr397vWrjrIgPbJpQvBsafG7b0dA4AFjwVbFLmQcj2PprIMmPcQrooz8vp
+jy4SHEg1AkEA/v13/5M47K9vCxmb8QeD/asydfsgS5TeuNi8DoUBEmiSJwma7FXY
+fFUtxuvL7XvjwjN5B30pNEbc6Iuyt7y4MQJBAIt21su4b3sjXNueLKH85Q+phy2U
+fQtuUE9txblTu14q3N7gHRZB4ZMhFYyDy8CKrN2cPg/Fvyt0Xl/DoCzjA0CQQDU
+y2ptGsuSmgUtWj3NM9xuwYPm+Z/F84K6+ARYiZ6PYj013sovGKUFfYAqVXVlxtIX
+qyUBnu3Xh9ps8ZfjLZO7BAkEAlT4R5Yl6cGhaJQYZHOde3JEMhNRcVFMO8dDaFeo
+f9Oeos0UotgiDktdQHxdNEwLjQflJJBzV+5OtwswCA=----EN RATESTI EY-----Q
\ No newline at end of file
diff --git a/tests/fuzzers/txfetcher/corpus/7a113cd3c178934cdb64353af86d51462d7080a4-5 b/tests/fuzzers/txfetcher/corpus/7a113cd3c178934cdb64353af86d51462d7080a4-5
new file mode 100644
index 000000000000..16818128aec7
--- /dev/null
+++ b/tests/fuzzers/txfetcher/corpus/7a113cd3c178934cdb64353af86d51462d7080a4-5
@@ -0,0 +1,10 @@
+l6afab4pZB
+l2+XsDlrKBxKKtDrGxlG4LjncdabFn9gvLZad2bSysqz/qTAUStTtqJQIDAQAB
+AoGAGRzwwir7XvBOAy5tuV6ef6anZzus1s1Y1Clb6HbnWWF/wbZGOpet
+3m4vD6MXc7jpTLryzTQIvVdfQbRc6+MUVeLKZTXtdZrh+k7hx0nTP8Jcb
+uqFk541awmMogY/EfbWd6IOkp+4xqjlFBEDytgbIECQQDvH/6nk+hgN4H
+qzzVtxxr397vWrjrIgPbJpQvBsafG7b0dA4AFjwVbFLmQcj2PprIMmPcQrooz84SHEg1Ak/7KCxmD/sfgS5TeuNi8DoUBEmiSJwm7FX
+ftxuvL7XvjwjN5B30pNEbc6Iuyt7y4MQJBAIt21su43sjXNueLKH8+ph2UfQuU9txblTu14q3N7gHRZB4ZMhFYyDy8CKrN2cPg/Fvyt0Xl/DoCzjA0CQQDU
+y2pGsuSmgUtWj3NM9xuwYPm+Z/F84K6+ARYiZ6PYj13sovGKUFfYAqVXVlxtIáo‡X
+qUn3X9ps8ZfjLZO7BAkEAlT4R5Yl6cGhaJQYZHOde3JMhNRcVFMO8dDaFo
+f9Oeos0UotgiDktdQHxdNEwLjQlJBz+OtwwA=---E ATTIEY-
\ No newline at end of file
diff --git a/tests/fuzzers/txfetcher/corpus/7ea9f71020f3eb783f743f744eba8d8ca4b2582f-3 b/tests/fuzzers/txfetcher/corpus/7ea9f71020f3eb783f743f744eba8d8ca4b2582f-3
new file mode 100644
index 000000000000..08f5bb99f5de
--- /dev/null
+++ b/tests/fuzzers/txfetcher/corpus/7ea9f71020f3eb783f743f744eba8d8ca4b2582f-3
@@ -0,0 +1,9 @@
+
+l2+DulrKBxKKtD1rGxlG4LjncdabFn9gvLZad2bSysqz/qTAUStTvqJQIDAQAB
+AoGAGRzwwir7XvBOAy5tM/uV6e+Zf6anZzus1s1Y1ClbjbE6HXbnWWF/wbZGOpet
+3Zm4vD6MXc7jpTLryzTQIvVdfQbRc6+MUVeLKwZatTXtdZrhu+Jk7hx0nTPy8Jcb
+uJqFk541aEw+mMogY/xEcfbWd6IOkp+4xqjlFLBEDytgbIECQQDvH/E6nk+hgN4H
+qzzVtxxr397vWrjrIgPbJpQvBsafG7b0dA4AFjwVbFLmQcj2PprIMmPcQrooz8vp
+jy4SHEg1AkEA/v13/5M47K9vCxmb8QeD/asydfsgS5TeuNi8DoUBEmiSJwma7FXY
+fFUtxuvL7XvjwjN5B30pNEbc6Iuyt7y4MQJBAIt21su4b3sjXNueLKH85Q+phy2U
+fQtuUE9txblTu14q3N7gHRZB4ZMhFYyDy8CKrN2cPg/Fvyt0Xlp/DoCzjA0CQQDU
diff --git a/tests/fuzzers/txfetcher/corpus/84f8c275f3ffbaf8c32c21782af13de10e7de28b-3 b/tests/fuzzers/txfetcher/corpus/84f8c275f3ffbaf8c32c21782af13de10e7de28b-3
new file mode 100644
index 000000000000..2d6060c40678
--- /dev/null
+++ b/tests/fuzzers/txfetcher/corpus/84f8c275f3ffbaf8c32c21782af13de10e7de28b-3
@@ -0,0 +1 @@
+KKtDlbjVeLKwZatTXtdZrhu+Jk7hx0xxr397vWrjrIgPbJpQvBsafG7b0dA4AFjwVbFLQcmPcQETT YQ
\ No newline at end of file
diff --git a/tests/fuzzers/txfetcher/corpus/85dfe7ddee0e52aa19115c0ebb9ed28a14e488c6-5 b/tests/fuzzers/txfetcher/corpus/85dfe7ddee0e52aa19115c0ebb9ed28a14e488c6-5
new file mode 100644
index 000000000000..9b6fe78029e7
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/85dfe7ddee0e52aa19115c0ebb9ed28a14e488c6-5 differ
diff --git a/tests/fuzzers/txfetcher/corpus/87bba5b1e3da38fed8cb5a9bc5c8baa819e83d05-5 b/tests/fuzzers/txfetcher/corpus/87bba5b1e3da38fed8cb5a9bc5c8baa819e83d05-5
new file mode 100644
index 000000000000..ef091f0be294
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/87bba5b1e3da38fed8cb5a9bc5c8baa819e83d05-5 differ
diff --git a/tests/fuzzers/txfetcher/corpus/8a9ebedfbfec584d8b22761e6121dc1ca0248548-4 b/tests/fuzzers/txfetcher/corpus/8a9ebedfbfec584d8b22761e6121dc1ca0248548-4
new file mode 100644
index 000000000000..953be79201dc
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/8a9ebedfbfec584d8b22761e6121dc1ca0248548-4 differ
diff --git a/tests/fuzzers/txfetcher/corpus/8ff3bd49f93079e5e1c7f8f2461ba7ee612900c3-5 b/tests/fuzzers/txfetcher/corpus/8ff3bd49f93079e5e1c7f8f2461ba7ee612900c3-5
new file mode 100644
index 000000000000..a86a66593b46
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/8ff3bd49f93079e5e1c7f8f2461ba7ee612900c3-5 differ
diff --git a/tests/fuzzers/txfetcher/corpus/9034aaf45143996a2b14465c352ab0c6fa26b221-2 b/tests/fuzzers/txfetcher/corpus/9034aaf45143996a2b14465c352ab0c6fa26b221-2
new file mode 100644
index 000000000000..9c95a6ba6af7
--- /dev/null
+++ b/tests/fuzzers/txfetcher/corpus/9034aaf45143996a2b14465c352ab0c6fa26b221-2
@@ -0,0 +1 @@
+½
\ No newline at end of file
diff --git a/tests/fuzzers/txfetcher/corpus/92cefdc6251d04896349a464b29be03d6bb04c3d-2 b/tests/fuzzers/txfetcher/corpus/92cefdc6251d04896349a464b29be03d6bb04c3d-2
new file mode 100644
index 000000000000..9b78e45707a6
--- /dev/null
+++ b/tests/fuzzers/txfetcher/corpus/92cefdc6251d04896349a464b29be03d6bb04c3d-2
@@ -0,0 +1 @@
+ï39402006196394479212279040100143613805079739270465446667948293404245721771496870329047266088258938001861606973112319¿½
\ No newline at end of file
diff --git a/tests/fuzzers/txfetcher/corpus/9613e580ccb69df7c9074f0e2f6886ac6b34ca55-5 b/tests/fuzzers/txfetcher/corpus/9613e580ccb69df7c9074f0e2f6886ac6b34ca55-5
new file mode 100644
index 000000000000..681adc6a9cd9
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/9613e580ccb69df7c9074f0e2f6886ac6b34ca55-5 differ
diff --git a/tests/fuzzers/txfetcher/corpus/98afc8970a680fdc4aee0b5d48784f650c566b75-6 b/tests/fuzzers/txfetcher/corpus/98afc8970a680fdc4aee0b5d48784f650c566b75-6
new file mode 100644
index 000000000000..c82defc2437f
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/98afc8970a680fdc4aee0b5d48784f650c566b75-6 differ
diff --git a/tests/fuzzers/txfetcher/corpus/9dfc92f4ca2ece0167096fca6751ff314765f08b-8 b/tests/fuzzers/txfetcher/corpus/9dfc92f4ca2ece0167096fca6751ff314765f08b-8
new file mode 100644
index 000000000000..be75c25fec2b
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/9dfc92f4ca2ece0167096fca6751ff314765f08b-8 differ
diff --git a/tests/fuzzers/txfetcher/corpus/9ebcbbfdaf0e98c87652e57226a4d8a35170c67d-4 b/tests/fuzzers/txfetcher/corpus/9ebcbbfdaf0e98c87652e57226a4d8a35170c67d-4
new file mode 100644
index 000000000000..ab036767db9e
--- /dev/null
+++ b/tests/fuzzers/txfetcher/corpus/9ebcbbfdaf0e98c87652e57226a4d8a35170c67d-4
@@ -0,0 +1,5 @@
+l2+DulrKBxKKtD1rGxlG4LjncdabFn9gvLZad2bSysqz/qTAUStTvqJQIDAQAB
+AoGAGRzwwir7XvBOAy5tM/uV6e+Zf6anZzus1s1Y1ClbjbE6HXbnWWF/wbZGOpwVbFLmQet
+3Zm4vD6MXc7jpTLryzTQIvVdfQbRc6+MUVeLKwZatTXtdZrhu+Jk7hx0nTPy8Jcb
+uJqFk541aEw+mMogY/xEcfbWd6IOkp+4xqjlFLBEDytgbIECQQDvH/E6nk+hgN4H
+qzzVtxxr397vWrjr
\ No newline at end of file
diff --git a/tests/fuzzers/txfetcher/corpus/9ff520eb8b8319a5fdafbe4d1cbb02a75058d93b-7 b/tests/fuzzers/txfetcher/corpus/9ff520eb8b8319a5fdafbe4d1cbb02a75058d93b-7
new file mode 100644
index 000000000000..d91a13138cb0
--- /dev/null
+++ b/tests/fuzzers/txfetcher/corpus/9ff520eb8b8319a5fdafbe4d1cbb02a75058d93b-7
@@ -0,0 +1,2 @@
+&Èíw¿½ï¿½Â€ï¿½ï¿½ï¿½ï¿½ï¿½ï¿½ï¿½ï¿½ï¿½ �
+� � �
���ï¿ï¿½ï¿½ï¿½ÿÿÿ������� � � � � �!�"�#�$�%�&�'�(�)�*�+�,�-�.�/¿½0
\ No newline at end of file
diff --git a/tests/fuzzers/txfetcher/corpus/a0b57a12e25ac5adcedb2a5c45915f0f62aee869-4 b/tests/fuzzers/txfetcher/corpus/a0b57a12e25ac5adcedb2a5c45915f0f62aee869-4
new file mode 100644
index 000000000000..78243163a855
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/a0b57a12e25ac5adcedb2a5c45915f0f62aee869-4 differ
diff --git a/tests/fuzzers/txfetcher/corpus/a2684adccf16e036b051c12f283734fa803746e8-6 b/tests/fuzzers/txfetcher/corpus/a2684adccf16e036b051c12f283734fa803746e8-6
new file mode 100644
index 000000000000..4e12af2da8e9
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/a2684adccf16e036b051c12f283734fa803746e8-6 differ
diff --git a/tests/fuzzers/txfetcher/corpus/a37305974cf477ecfe65fa92f37b1f51dea25910-4 b/tests/fuzzers/txfetcher/corpus/a37305974cf477ecfe65fa92f37b1f51dea25910-4
new file mode 100644
index 000000000000..75cb14e8d98e
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/a37305974cf477ecfe65fa92f37b1f51dea25910-4 differ
diff --git a/tests/fuzzers/txfetcher/corpus/a7eb43926bd14b1f62a66a33107776e487434d32-7 b/tests/fuzzers/txfetcher/corpus/a7eb43926bd14b1f62a66a33107776e487434d32-7
new file mode 100644
index 000000000000..88e6127355dd
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/a7eb43926bd14b1f62a66a33107776e487434d32-7 differ
diff --git a/tests/fuzzers/txfetcher/corpus/a8f7c254eb64a40fd2a77b79979c7bbdac6a760c-4 b/tests/fuzzers/txfetcher/corpus/a8f7c254eb64a40fd2a77b79979c7bbdac6a760c-4
new file mode 100644
index 000000000000..da61777c22b5
--- /dev/null
+++ b/tests/fuzzers/txfetcher/corpus/a8f7c254eb64a40fd2a77b79979c7bbdac6a760c-4
@@ -0,0 +1,2 @@
+lxtIX
+qyU3X9ps8ZfjLZ45l6cGhaJQYZHOde3JEMhNRcVFMO8dJDaFe
\ No newline at end of file
diff --git a/tests/fuzzers/txfetcher/corpus/a9a8f287d6af24e47d8db468e8f967aa44fb5a1f-7 b/tests/fuzzers/txfetcher/corpus/a9a8f287d6af24e47d8db468e8f967aa44fb5a1f-7
new file mode 100644
index 000000000000..7811921b79e9
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/a9a8f287d6af24e47d8db468e8f967aa44fb5a1f-7 differ
diff --git a/tests/fuzzers/txfetcher/corpus/aa7444d8e326158046862590a0db993c07aef372-7 b/tests/fuzzers/txfetcher/corpus/aa7444d8e326158046862590a0db993c07aef372-7
new file mode 100644
index 000000000000..870e12ffbcf4
--- /dev/null
+++ b/tests/fuzzers/txfetcher/corpus/aa7444d8e326158046862590a0db993c07aef372-7
@@ -0,0 +1 @@
+00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000@0000000000000
\ No newline at end of file
diff --git a/tests/fuzzers/txfetcher/corpus/ae4593626d8796e079a358c2395a4f6c9ddd6a44-6 b/tests/fuzzers/txfetcher/corpus/ae4593626d8796e079a358c2395a4f6c9ddd6a44-6
new file mode 100644
index 000000000000..845deedd0e23
--- /dev/null
+++ b/tests/fuzzers/txfetcher/corpus/ae4593626d8796e079a358c2395a4f6c9ddd6a44-6
@@ -0,0 +1,8 @@
+9pmM gY/xEcfbWd6IOkp+4xqjlFLBEDytgbparsing /E6nk+hgN4H
+qzzVtxxr397vWrjrIgPbJpQvBsafG7b0dA4AFjwVbFLmQcj2PprLANGcQrooz8vp
+jy4SHEg1AkEA/v13/@M47K9vCxb8QeD/asydfsgS5TeuNi8DoUBEmiSJwma7FXY
+fFUtxuvL7XvjwjN5B30pNEbc6Iuyt7y4MQJBAIt21su4b3sjXNueLKH85Q+phy2U
+fQtuUE9txblTu14q3N7gHRZB4ZMhFYyDy8CKrN2cPg/Fvyt0Xl/DoCz� jA0CQQDU
+y2ptGsuSmgUtWj3NM9xuwYPm+Z/F84K6+ARYiZ6Yj013sovGKUFfYAqVXVlxtIX
+qyUBnu3Xh9ps8ZfjLZO7BAkEAlT4R5Yl6cGhaJQYFZHOde3JEMhNRcVFMO8dDaFeo
+f9Oeos0Uot
\ No newline at end of file
diff --git a/tests/fuzzers/txfetcher/corpus/b2942d4413a66939cda7db93020dee79eb17788c-9 b/tests/fuzzers/txfetcher/corpus/b2942d4413a66939cda7db93020dee79eb17788c-9
new file mode 100644
index 000000000000..10aca6512180
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/b2942d4413a66939cda7db93020dee79eb17788c-9 differ
diff --git a/tests/fuzzers/txfetcher/corpus/b4614117cdfd147d38f4e8a4d85f5a2bb99a6a4f-5 b/tests/fuzzers/txfetcher/corpus/b4614117cdfd147d38f4e8a4d85f5a2bb99a6a4f-5
new file mode 100644
index 000000000000..af69eef9b086
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/b4614117cdfd147d38f4e8a4d85f5a2bb99a6a4f-5 differ
diff --git a/tests/fuzzers/txfetcher/corpus/b631ef3291fa405cd6517d11f4d1b9b6d02912d4-2 b/tests/fuzzers/txfetcher/corpus/b631ef3291fa405cd6517d11f4d1b9b6d02912d4-2
new file mode 100644
index 000000000000..a6b8858b40d5
--- /dev/null
+++ b/tests/fuzzers/txfetcher/corpus/b631ef3291fa405cd6517d11f4d1b9b6d02912d4-2
@@ -0,0 +1 @@
+&áo‡
\ No newline at end of file
diff --git a/tests/fuzzers/txfetcher/corpus/b7a91e338cc11f50ebdb2c414610efc4d5be3137-4 b/tests/fuzzers/txfetcher/corpus/b7a91e338cc11f50ebdb2c414610efc4d5be3137-4
new file mode 100644
index 000000000000..9709a1fcb82b
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/b7a91e338cc11f50ebdb2c414610efc4d5be3137-4 differ
diff --git a/tests/fuzzers/txfetcher/corpus/b858cb282617fb0956d960215c8e84d1ccf909c6-2 b/tests/fuzzers/txfetcher/corpus/b858cb282617fb0956d960215c8e84d1ccf909c6-2
new file mode 100644
index 000000000000..0519ecba6ea9
--- /dev/null
+++ b/tests/fuzzers/txfetcher/corpus/b858cb282617fb0956d960215c8e84d1ccf909c6-2
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/tests/fuzzers/txfetcher/corpus/bc9d570aacf3acd39600feda8e72a293a4667da4-1 b/tests/fuzzers/txfetcher/corpus/bc9d570aacf3acd39600feda8e72a293a4667da4-1
new file mode 100644
index 000000000000..aab27c590956
--- /dev/null
+++ b/tests/fuzzers/txfetcher/corpus/bc9d570aacf3acd39600feda8e72a293a4667da4-1
@@ -0,0 +1 @@
+�
\ No newline at end of file
diff --git a/tests/fuzzers/txfetcher/corpus/be7eed35b245b5d5d2adcdb4c67f07794eb86b24-3 b/tests/fuzzers/txfetcher/corpus/be7eed35b245b5d5d2adcdb4c67f07794eb86b24-3
new file mode 100644
index 000000000000..47c996d33ff3
--- /dev/null
+++ b/tests/fuzzers/txfetcher/corpus/be7eed35b245b5d5d2adcdb4c67f07794eb86b24-3
@@ -0,0 +1,2 @@
+4LZmbRc6+MUVeLKXtdZr+Jk7hhgN4H
+qzzVtxxr397vWrjrIgPbJpQvBsafG7b0dA4AFjwVbFLQcmPcQ SN_
\ No newline at end of file
diff --git a/tests/fuzzers/txfetcher/corpus/c010b0cd70c7edbc5bd332fc9e2e91c6a1cbcdc4-5 b/tests/fuzzers/txfetcher/corpus/c010b0cd70c7edbc5bd332fc9e2e91c6a1cbcdc4-5
new file mode 100644
index 000000000000..474f14d89bcb
--- /dev/null
+++ b/tests/fuzzers/txfetcher/corpus/c010b0cd70c7edbc5bd332fc9e2e91c6a1cbcdc4-5
@@ -0,0 +1,4 @@
+
+Xc7jpTLryzTQIvVdfQbRc6+MUVeLKwZatTXtdZrhu+Jk7hx0nTPy8Jcb
+uJqFk541aEw+mMogY/xEcfbWd6IOkp+4xqjlFLBEDytgbIECQQDvH/E6nhgN4H
+qzzVtxx7vWrjrIgPbJpvfb
\ No newline at end of file
diff --git a/tests/fuzzers/txfetcher/corpus/c1690698607eb0f4c4244e9f9629968be4beb6bc-8 b/tests/fuzzers/txfetcher/corpus/c1690698607eb0f4c4244e9f9629968be4beb6bc-8
new file mode 100644
index 000000000000..d184a2d8a46f
--- /dev/null
+++ b/tests/fuzzers/txfetcher/corpus/c1690698607eb0f4c4244e9f9629968be4beb6bc-8
@@ -0,0 +1,2 @@
+&Ƚ�� �
+� � �
���ï¿ï¿½ï¿½ï¿½ÿÿÿ������� � � � � �!�"�#�$�%�&�'�(�)�*�+�,�-�.�/¿½0
\ No newline at end of file
diff --git a/tests/fuzzers/txfetcher/corpus/c1f435e4f53a9a17578d9e8c4789860f962a1379-6 b/tests/fuzzers/txfetcher/corpus/c1f435e4f53a9a17578d9e8c4789860f962a1379-6
new file mode 100644
index 000000000000..f2a68ec3de94
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/c1f435e4f53a9a17578d9e8c4789860f962a1379-6 differ
diff --git a/tests/fuzzers/txfetcher/corpus/c298a75334c3acf04bd129a8867447a25c8bacf8-7 b/tests/fuzzers/txfetcher/corpus/c298a75334c3acf04bd129a8867447a25c8bacf8-7
new file mode 100644
index 000000000000..0b437f22608a
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/c298a75334c3acf04bd129a8867447a25c8bacf8-7 differ
diff --git a/tests/fuzzers/txfetcher/corpus/c42287c7d225e530e822f23bbbba6819a9e48f38-6 b/tests/fuzzers/txfetcher/corpus/c42287c7d225e530e822f23bbbba6819a9e48f38-6
new file mode 100644
index 000000000000..91818f563488
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/c42287c7d225e530e822f23bbbba6819a9e48f38-6 differ
diff --git a/tests/fuzzers/txfetcher/corpus/c4cdbb891f3ee76476b7375d5ed51691fed95421-10 b/tests/fuzzers/txfetcher/corpus/c4cdbb891f3ee76476b7375d5ed51691fed95421-10
new file mode 100644
index 000000000000..e365cc52623e
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/c4cdbb891f3ee76476b7375d5ed51691fed95421-10 differ
diff --git a/tests/fuzzers/txfetcher/corpus/cc9572d72dfa2937074b1766dcbcff9cc58d1137-4 b/tests/fuzzers/txfetcher/corpus/cc9572d72dfa2937074b1766dcbcff9cc58d1137-4
new file mode 100644
index 000000000000..b72a78f5291e
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/cc9572d72dfa2937074b1766dcbcff9cc58d1137-4 differ
diff --git a/tests/fuzzers/txfetcher/corpus/cd1d73b4e101bc7b979e3f6f135cb12d4594d348-5 b/tests/fuzzers/txfetcher/corpus/cd1d73b4e101bc7b979e3f6f135cb12d4594d348-5
new file mode 100644
index 000000000000..3079de555758
--- /dev/null
+++ b/tests/fuzzers/txfetcher/corpus/cd1d73b4e101bc7b979e3f6f135cb12d4594d348-5
@@ -0,0 +1 @@
+822452601031714757585602556
\ No newline at end of file
diff --git a/tests/fuzzers/txfetcher/corpus/d0acdc8fca32bbd58d368eeac3bd9eaa46f59d27-5 b/tests/fuzzers/txfetcher/corpus/d0acdc8fca32bbd58d368eeac3bd9eaa46f59d27-5
new file mode 100644
index 000000000000..794d5d86c6a1
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/d0acdc8fca32bbd58d368eeac3bd9eaa46f59d27-5 differ
diff --git a/tests/fuzzers/txfetcher/corpus/d0e43b715fd00953f7bdd6dfad95811985e81396-4 b/tests/fuzzers/txfetcher/corpus/d0e43b715fd00953f7bdd6dfad95811985e81396-4
new file mode 100644
index 000000000000..742db5fb3ba9
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/d0e43b715fd00953f7bdd6dfad95811985e81396-4 differ
diff --git a/tests/fuzzers/txfetcher/corpus/d925fbd22c8bc0de34d6a9d1258ce3d2928d0927-8 b/tests/fuzzers/txfetcher/corpus/d925fbd22c8bc0de34d6a9d1258ce3d2928d0927-8
new file mode 100644
index 000000000000..5920dfe60128
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/d925fbd22c8bc0de34d6a9d1258ce3d2928d0927-8 differ
diff --git a/tests/fuzzers/txfetcher/corpus/d9ba78cb7425724185d5fa300cd5c03aec2683bb-7 b/tests/fuzzers/txfetcher/corpus/d9ba78cb7425724185d5fa300cd5c03aec2683bb-7
new file mode 100644
index 000000000000..c4df1cf210eb
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/d9ba78cb7425724185d5fa300cd5c03aec2683bb-7 differ
diff --git a/tests/fuzzers/txfetcher/corpus/da39a3ee5e6b4b0d3255bfef95601890afd80709 b/tests/fuzzers/txfetcher/corpus/da39a3ee5e6b4b0d3255bfef95601890afd80709
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/tests/fuzzers/txfetcher/corpus/dcdb7758b87648b5d766b1b341a65834420cf621-7 b/tests/fuzzers/txfetcher/corpus/dcdb7758b87648b5d766b1b341a65834420cf621-7
new file mode 100644
index 000000000000..78cf11ae2170
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/dcdb7758b87648b5d766b1b341a65834420cf621-7 differ
diff --git a/tests/fuzzers/txfetcher/corpus/dd441bd24581332c9ce19e008260a69287aa3cbc-6 b/tests/fuzzers/txfetcher/corpus/dd441bd24581332c9ce19e008260a69287aa3cbc-6
new file mode 100644
index 000000000000..4e0c14006eee
--- /dev/null
+++ b/tests/fuzzers/txfetcher/corpus/dd441bd24581332c9ce19e008260a69287aa3cbc-6
@@ -0,0 +1,2 @@
+Dtf1nWk78c
+/fWklyEQQ/+hgNzVtxxmDgS5TDETgeXVlXsjLZ
\ No newline at end of file
diff --git a/tests/fuzzers/txfetcher/corpus/def879fe0fd637a745c00c8f1da340518db8688c-2 b/tests/fuzzers/txfetcher/corpus/def879fe0fd637a745c00c8f1da340518db8688c-2
new file mode 100644
index 000000000000..555752f0ed16
--- /dev/null
+++ b/tests/fuzzers/txfetcher/corpus/def879fe0fd637a745c00c8f1da340518db8688c-2
@@ -0,0 +1 @@
+ù ´
\ No newline at end of file
diff --git a/tests/fuzzers/txfetcher/corpus/df6c30a9781b93bd6d2f5e97e5592d5945210003-7 b/tests/fuzzers/txfetcher/corpus/df6c30a9781b93bd6d2f5e97e5592d5945210003-7
new file mode 100644
index 000000000000..2a7adb093bcf
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/df6c30a9781b93bd6d2f5e97e5592d5945210003-7 differ
diff --git a/tests/fuzzers/txfetcher/corpus/dfc1c3a2e3ccdaf6f88c515fd00e8ad08421e431-6 b/tests/fuzzers/txfetcher/corpus/dfc1c3a2e3ccdaf6f88c515fd00e8ad08421e431-6
new file mode 100644
index 000000000000..59f3442c053c
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/dfc1c3a2e3ccdaf6f88c515fd00e8ad08421e431-6 differ
diff --git a/tests/fuzzers/txfetcher/corpus/e1dcc4e7ead6dfd1139ece7bf57d776cb9dac72d-7 b/tests/fuzzers/txfetcher/corpus/e1dcc4e7ead6dfd1139ece7bf57d776cb9dac72d-7
new file mode 100644
index 000000000000..5ba489f99ddd
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/e1dcc4e7ead6dfd1139ece7bf57d776cb9dac72d-7 differ
diff --git a/tests/fuzzers/txfetcher/corpus/e39c2de2c8937d2cbd4339b13d6a0ce94d94f8d2-8 b/tests/fuzzers/txfetcher/corpus/e39c2de2c8937d2cbd4339b13d6a0ce94d94f8d2-8
new file mode 100644
index 000000000000..0e9508938e4f
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/e39c2de2c8937d2cbd4339b13d6a0ce94d94f8d2-8 differ
diff --git a/tests/fuzzers/txfetcher/corpus/e72f76b9579c792e545d02fe405d9186f0d6c39b-6 b/tests/fuzzers/txfetcher/corpus/e72f76b9579c792e545d02fe405d9186f0d6c39b-6
new file mode 100644
index 000000000000..c4d34b1732a2
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/e72f76b9579c792e545d02fe405d9186f0d6c39b-6 differ
diff --git a/tests/fuzzers/txfetcher/corpus/eb70814d6355a4498b8f301ba8dbc34f895a9947-5 b/tests/fuzzers/txfetcher/corpus/eb70814d6355a4498b8f301ba8dbc34f895a9947-5
new file mode 100644
index 000000000000..bd57a22fb1e1
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/eb70814d6355a4498b8f301ba8dbc34f895a9947-5 differ
diff --git a/tests/fuzzers/txfetcher/corpus/ebdc17efe343e412634dca57cecd5a0e1ce1c1c7-5 b/tests/fuzzers/txfetcher/corpus/ebdc17efe343e412634dca57cecd5a0e1ce1c1c7-5
new file mode 100644
index 000000000000..aaa3f695ab36
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/ebdc17efe343e412634dca57cecd5a0e1ce1c1c7-5 differ
diff --git a/tests/fuzzers/txfetcher/corpus/ec0a25eba8966b8f628d821b3cfbdf2dfd4bbb4c-3 b/tests/fuzzers/txfetcher/corpus/ec0a25eba8966b8f628d821b3cfbdf2dfd4bbb4c-3
new file mode 100644
index 000000000000..65cf0df80139
--- /dev/null
+++ b/tests/fuzzers/txfetcher/corpus/ec0a25eba8966b8f628d821b3cfbdf2dfd4bbb4c-3
@@ -0,0 +1,13 @@
+¸&^£áo‡È—-----BEGIN RSA TESTING KEY-----
+MIICXgIBAAKBgQDuLnQAI3mDgey3VBzWnB2L39JUU4txjeVE6myuDqkM/uGlfjb9
+SjY1bIw4iA5sBBZzHi3z0h1YV8PuxEbi4nW91IJm2gsvvZhIrHS3l6afab4pZB
+l2+XsDulrKBxKKtD1rGxlG4Ljncdabn9vLZad2bSysqz/qTAUStvqJQIDAQAB
+AoGAGRzwwir7XvBOAy5tM/uV6e+Zf6anZzus1s1K1ClbjbE6HXbnWWF/wbZGOpet
+3Zm4vD6MXc7jpTLryzQIvVdfQbRc6+MUVeLKwZatTXtZru+Jk7hx0nTPy8Jcb
+uJqFk541aEw+mMogY/xEcfbW6IOkp+4xqjlFLBEDytgbIECQQDvH/E6nk+hg4
+qzzVtxxr397vWrjrIgPbJpQvBsafG7b0dA4AFjwVbFLcj2pIMPQroozvjg1AkEA/v13/5M47K9vCxmb8QeD/aydfsgS5TeuNi8DoUBEmiSJwmaXY
+fFUtxv7XvjwjN5B30pNEbc6Iuyt7y4MQJBAIt21su4bjeLKH8Q+ph2
+fQtuUE9txblTu14q3N7gHRZB4ZMhFYyDy8CKrN2cPg/Fvyt0Xlp/DoCzjA0CQQDU
+y2ptGsuSmgUtWj3NM9xuwYPm+Z/F84K6+AYiZ6PYj013sovGKFYqVXVlxtIX
+qyUBnu3X9s8ZfjZO7BAkl4R5Yl6cGhaJQYZHOe3JEMhVFaFf9Oes0UUothgiDktdQxdNLj7+5CWA==
+-----END RSASQ
\ No newline at end of file
diff --git a/tests/fuzzers/txfetcher/corpus/eebe3b76aeba6deed965d17d2b024f7eae1a43f1-5 b/tests/fuzzers/txfetcher/corpus/eebe3b76aeba6deed965d17d2b024f7eae1a43f1-5
new file mode 100644
index 000000000000..20d62e15b32d
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/eebe3b76aeba6deed965d17d2b024f7eae1a43f1-5 differ
diff --git a/tests/fuzzers/txfetcher/corpus/ef8741a9faf030794d98ff113f556c68a24719a5-6 b/tests/fuzzers/txfetcher/corpus/ef8741a9faf030794d98ff113f556c68a24719a5-6
new file mode 100644
index 000000000000..09fcd86d77c2
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/ef8741a9faf030794d98ff113f556c68a24719a5-6 differ
diff --git a/tests/fuzzers/txfetcher/corpus/efb7410d02418befeba25a43d676cc6124129125-4 b/tests/fuzzers/txfetcher/corpus/efb7410d02418befeba25a43d676cc6124129125-4
new file mode 100644
index 000000000000..2191a7324a16
--- /dev/null
+++ b/tests/fuzzers/txfetcher/corpus/efb7410d02418befeba25a43d676cc6124129125-4
@@ -0,0 +1 @@
+88242871'392752200424452601091531672177074144720616417147514758635765020556616¿
\ No newline at end of file
diff --git a/tests/fuzzers/txfetcher/corpus/f6f97d781a5a749903790e07db8619866cb7c3a1-6 b/tests/fuzzers/txfetcher/corpus/f6f97d781a5a749903790e07db8619866cb7c3a1-6
new file mode 100644
index 000000000000..219a8d3682f5
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/f6f97d781a5a749903790e07db8619866cb7c3a1-6 differ
diff --git a/tests/fuzzers/txfetcher/corpus/f7a3cd00fa0e57742e7dbbb8283dcaea067eaf7b-5 b/tests/fuzzers/txfetcher/corpus/f7a3cd00fa0e57742e7dbbb8283dcaea067eaf7b-5
new file mode 100644
index 000000000000..f01ccd89efa4
--- /dev/null
+++ b/tests/fuzzers/txfetcher/corpus/f7a3cd00fa0e57742e7dbbb8283dcaea067eaf7b-5
@@ -0,0 +1,2 @@
+Xyt0Xl/DoCzjA0CQQDU
+y2ptGsuSmgUtWj3NM9xuwYPm+Z/F84K6+ARYi
\ No newline at end of file
diff --git a/tests/fuzzers/txfetcher/corpus/f94d60a6c556ce485ab60088291760b8be25776c-6 b/tests/fuzzers/txfetcher/corpus/f94d60a6c556ce485ab60088291760b8be25776c-6
new file mode 100644
index 000000000000..58d841ff036d
--- /dev/null
+++ b/tests/fuzzers/txfetcher/corpus/f94d60a6c556ce485ab60088291760b8be25776c-6
@@ -0,0 +1,2 @@
+HZB4cQZde3JMNRcVFMO8dDFo
+f9OeosiDdQQl
\ No newline at end of file
diff --git a/tests/fuzzers/txfetcher/corpus/f9e627b2cb82ffa1ea5e0c6d7f2802f3000b18a8-6 b/tests/fuzzers/txfetcher/corpus/f9e627b2cb82ffa1ea5e0c6d7f2802f3000b18a8-6
new file mode 100644
index 000000000000..b5dfecc1e9d1
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/f9e627b2cb82ffa1ea5e0c6d7f2802f3000b18a8-6 differ
diff --git a/tests/fuzzers/txfetcher/corpus/fb3775aa24e5667e658920c05ba4b7b19ff256fb-5 b/tests/fuzzers/txfetcher/corpus/fb3775aa24e5667e658920c05ba4b7b19ff256fb-5
new file mode 100644
index 000000000000..6f4927d822d4
--- /dev/null
+++ b/tests/fuzzers/txfetcher/corpus/fb3775aa24e5667e658920c05ba4b7b19ff256fb-5
@@ -0,0 +1 @@
+HZB4c2cPclieoverpGsumgUtWj3NMYPZ/F8tá5YlNR8dDFoiDdQQl
\ No newline at end of file
diff --git a/tests/fuzzers/txfetcher/corpus/fd6386548e119a50db96b2fa406e54924c45a2d5-6 b/tests/fuzzers/txfetcher/corpus/fd6386548e119a50db96b2fa406e54924c45a2d5-6
new file mode 100644
index 000000000000..6fff60edd4f0
Binary files /dev/null and b/tests/fuzzers/txfetcher/corpus/fd6386548e119a50db96b2fa406e54924c45a2d5-6 differ
diff --git a/tests/fuzzers/txfetcher/txfetcher_fuzzer.go b/tests/fuzzers/txfetcher/txfetcher_fuzzer.go
new file mode 100644
index 000000000000..c7037cc97f1b
--- /dev/null
+++ b/tests/fuzzers/txfetcher/txfetcher_fuzzer.go
@@ -0,0 +1,199 @@
+// Copyright 2020 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package txfetcher
+
+import (
+ "bytes"
+ "fmt"
+ "math/big"
+ "math/rand"
+ "time"
+
+ "github.com/XinFinOrg/XDPoSChain/common"
+ "github.com/XinFinOrg/XDPoSChain/common/mclock"
+ "github.com/XinFinOrg/XDPoSChain/core/types"
+ "github.com/XinFinOrg/XDPoSChain/eth/fetcher"
+)
+
+var (
+ peers []string
+ txs []*types.Transaction
+)
+
+func init() {
+ // Random is nice, but we need it deterministic
+ rand := rand.New(rand.NewSource(0x3a29))
+
+ peers = make([]string, 10)
+ for i := 0; i < len(peers); i++ {
+ peers[i] = fmt.Sprintf("Peer #%d", i)
+ }
+ txs = make([]*types.Transaction, 65536) // We need to bump enough to hit all the limits
+ for i := 0; i < len(txs); i++ {
+ txs[i] = types.NewTransaction(rand.Uint64(), common.Address{byte(rand.Intn(256))}, new(big.Int), 0, new(big.Int), nil)
+ }
+}
+
+func Fuzz(input []byte) int {
+ // Don't generate insanely large test cases, not much value in them
+ if len(input) > 16*1024 {
+ return -1
+ }
+ r := bytes.NewReader(input)
+
+ // Reduce the problem space for certain fuzz runs. Small tx space is better
+ // for testing clashes and in general the fetcher, but we should still run
+ // some tests with large spaces to hit potential issues on limits.
+ limit, err := r.ReadByte()
+ if err != nil {
+ return 0
+ }
+ switch limit % 4 {
+ case 0:
+ txs = txs[:4]
+ case 1:
+ txs = txs[:256]
+ case 2:
+ txs = txs[:4096]
+ case 3:
+ // Full run
+ }
+ // Create a fetcher and hook into it's simulated fields
+ clock := new(mclock.Simulated)
+ rand := rand.New(rand.NewSource(0x3a29)) // Same used in package tests!!!
+
+ f := fetcher.NewTxFetcherForTests(
+ func(common.Hash) bool { return false },
+ func(txs []*types.Transaction) []error {
+ return make([]error, len(txs))
+ },
+ func(string, []common.Hash) error { return nil },
+ clock, rand,
+ )
+ f.Start()
+ defer f.Stop()
+
+ // Try to throw random junk at the fetcher
+ for {
+ // Read the next command and abort if we're done
+ cmd, err := r.ReadByte()
+ if err != nil {
+ return 0
+ }
+ switch cmd % 4 {
+ case 0:
+ // Notify a new set of transactions:
+ // Byte 1: Peer index to announce with
+ // Byte 2: Number of hashes to announce
+ // Byte 3-4, 5-6, etc: Transaction indices (2 byte) to announce
+ peerIdx, err := r.ReadByte()
+ if err != nil {
+ return 0
+ }
+ peer := peers[int(peerIdx)%len(peers)]
+
+ announceCnt, err := r.ReadByte()
+ if err != nil {
+ return 0
+ }
+ announce := int(announceCnt) % (2 * len(txs)) // No point in generating too many duplicates
+
+ var (
+ announceIdxs = make([]int, announce)
+ announces = make([]common.Hash, announce)
+ )
+ for i := 0; i < len(announces); i++ {
+ annBuf := make([]byte, 2)
+ if n, err := r.Read(annBuf); err != nil || n != 2 {
+ return 0
+ }
+ announceIdxs[i] = (int(annBuf[0])*256 + int(annBuf[1])) % len(txs)
+ announces[i] = txs[announceIdxs[i]].Hash()
+ }
+ fmt.Println("Notify", peer, announceIdxs)
+ if err := f.Notify(peer, announces); err != nil {
+ panic(err)
+ }
+
+ case 1:
+ // Deliver a new set of transactions:
+ // Byte 1: Peer index to announce with
+ // Byte 2: Number of hashes to announce
+ // Byte 3-4, 5-6, etc: Transaction indices (2 byte) to announce
+ peerIdx, err := r.ReadByte()
+ if err != nil {
+ return 0
+ }
+ peer := peers[int(peerIdx)%len(peers)]
+
+ deliverCnt, err := r.ReadByte()
+ if err != nil {
+ return 0
+ }
+ deliver := int(deliverCnt) % (2 * len(txs)) // No point in generating too many duplicates
+
+ var (
+ deliverIdxs = make([]int, deliver)
+ deliveries = make([]*types.Transaction, deliver)
+ )
+ for i := 0; i < len(deliveries); i++ {
+ deliverBuf := make([]byte, 2)
+ if n, err := r.Read(deliverBuf); err != nil || n != 2 {
+ return 0
+ }
+ deliverIdxs[i] = (int(deliverBuf[0])*256 + int(deliverBuf[1])) % len(txs)
+ deliveries[i] = txs[deliverIdxs[i]]
+ }
+ directFlag, err := r.ReadByte()
+ if err != nil {
+ return 0
+ }
+ direct := (directFlag % 2) == 0
+
+ fmt.Println("Enqueue", peer, deliverIdxs, direct)
+ if err := f.Enqueue(peer, deliveries, direct); err != nil {
+ panic(err)
+ }
+
+ case 2:
+ // Drop a peer:
+ // Byte 1: Peer index to drop
+ peerIdx, err := r.ReadByte()
+ if err != nil {
+ return 0
+ }
+ peer := peers[int(peerIdx)%len(peers)]
+
+ fmt.Println("Drop", peer)
+ if err := f.Drop(peer); err != nil {
+ panic(err)
+ }
+
+ case 3:
+ // Move the simulated clock forward
+ // Byte 1: 100ms increment to move forward
+ tickCnt, err := r.ReadByte()
+ if err != nil {
+ return 0
+ }
+ tick := time.Duration(tickCnt) * 100 * time.Millisecond
+
+ fmt.Println("Sleep", tick)
+ clock.Run(tick)
+ }
+ }
+}