aboutsummaryrefslogtreecommitdiffstats
path: root/eth/downloader
diff options
context:
space:
mode:
authorPéter Szilágyi <peterke@gmail.com>2015-10-13 17:04:25 +0800
committerPéter Szilágyi <peterke@gmail.com>2015-10-21 21:49:55 +0800
commit5b0ee8ec304663898073b7a4c659e1def23716df (patch)
tree8f2f49a8d26dc1c29e1d360fb787ab420d90a2ae /eth/downloader
parentaa0538db0b5de2bb2c609d629b65d083649f9171 (diff)
downloadgo-tangerine-5b0ee8ec304663898073b7a4c659e1def23716df.tar.gz
go-tangerine-5b0ee8ec304663898073b7a4c659e1def23716df.tar.zst
go-tangerine-5b0ee8ec304663898073b7a4c659e1def23716df.zip
core, eth, trie: fix data races and merge/review issues
Diffstat (limited to 'eth/downloader')
-rw-r--r--eth/downloader/downloader.go191
-rw-r--r--eth/downloader/downloader_test.go464
-rw-r--r--eth/downloader/modes.go4
-rw-r--r--eth/downloader/peer.go28
-rw-r--r--eth/downloader/queue.go178
5 files changed, 518 insertions, 347 deletions
diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go
index 0298dfa0b..4bcbd8557 100644
--- a/eth/downloader/downloader.go
+++ b/eth/downloader/downloader.go
@@ -18,7 +18,9 @@
package downloader
import (
+ "crypto/rand"
"errors"
+ "fmt"
"math"
"math/big"
"strings"
@@ -59,9 +61,11 @@ var (
maxQueuedStates = 256 * 1024 // [eth/63] Maximum number of state requests to queue (DOS protection)
maxResultsProcess = 256 // Number of download results to import at once into the chain
- headerCheckFrequency = 100 // Verification frequency of the downloaded headers during fast sync
- minCheckedHeaders = 2048 // Number of headers to verify fully when approaching the chain head
- minFullBlocks = 1024 // Number of blocks to retrieve fully even in fast sync
+ fsHeaderCheckFrequency = 100 // Verification frequency of the downloaded headers during fast sync
+ fsHeaderSafetyNet = 2048 // Number of headers to discard in case a chain violation is detected
+ fsHeaderForceVerify = 24 // Number of headers to verify before and after the pivot to accept it
+ fsPivotInterval = 512 // Number of headers out of which to randomize the pivot point
+ fsMinFullBlocks = 1024 // Number of blocks to retrieve fully even in fast sync
)
var (
@@ -85,12 +89,14 @@ var (
errCancelHeaderFetch = errors.New("block header download canceled (requested)")
errCancelBodyFetch = errors.New("block body download canceled (requested)")
errCancelReceiptFetch = errors.New("receipt download canceled (requested)")
+ errCancelStateFetch = errors.New("state data download canceled (requested)")
errNoSyncActive = errors.New("no sync active")
)
type Downloader struct {
- mode SyncMode // Synchronisation mode defining the strategies used
- mux *event.TypeMux // Event multiplexer to announce sync operation events
+ mode SyncMode // Synchronisation mode defining the strategy used (per sync cycle)
+ noFast bool // Flag to disable fast syncing in case of a security error
+ mux *event.TypeMux // Event multiplexer to announce sync operation events
queue *queue // Scheduler for selecting the hashes to download
peers *peerSet // Set of active peers from which download can proceed
@@ -150,13 +156,13 @@ type Downloader struct {
}
// New creates a new downloader to fetch hashes and blocks from remote peers.
-func New(mode SyncMode, stateDb ethdb.Database, mux *event.TypeMux, hasHeader headerCheckFn, hasBlock blockCheckFn, getHeader headerRetrievalFn,
+func New(stateDb ethdb.Database, mux *event.TypeMux, hasHeader headerCheckFn, hasBlock blockCheckFn, getHeader headerRetrievalFn,
getBlock blockRetrievalFn, headHeader headHeaderRetrievalFn, headBlock headBlockRetrievalFn, headFastBlock headFastBlockRetrievalFn,
commitHeadBlock headBlockCommitterFn, getTd tdRetrievalFn, insertHeaders headerChainInsertFn, insertBlocks blockChainInsertFn,
insertReceipts receiptChainInsertFn, rollback chainRollbackFn, dropPeer peerDropFn) *Downloader {
return &Downloader{
- mode: mode,
+ mode: FullSync,
mux: mux,
queue: newQueue(stateDb),
peers: newPeerSet(),
@@ -188,19 +194,28 @@ func New(mode SyncMode, stateDb ethdb.Database, mux *event.TypeMux, hasHeader he
}
}
-// Boundaries retrieves the synchronisation boundaries, specifically the origin
-// block where synchronisation started at (may have failed/suspended) and the
-// latest known block which the synchonisation targets.
-func (d *Downloader) Boundaries() (uint64, uint64) {
+// Progress retrieves the synchronisation boundaries, specifically the origin
+// block where synchronisation started at (may have failed/suspended); the block
+// or header sync is currently at; and the latest known block which the sync targets.
+func (d *Downloader) Progress() (uint64, uint64, uint64) {
d.syncStatsLock.RLock()
defer d.syncStatsLock.RUnlock()
- return d.syncStatsChainOrigin, d.syncStatsChainHeight
+ current := uint64(0)
+ switch d.mode {
+ case FullSync:
+ current = d.headBlock().NumberU64()
+ case FastSync:
+ current = d.headFastBlock().NumberU64()
+ case LightSync:
+ current = d.headHeader().Number.Uint64()
+ }
+ return d.syncStatsChainOrigin, current, d.syncStatsChainHeight
}
// Synchronising returns whether the downloader is currently retrieving blocks.
func (d *Downloader) Synchronising() bool {
- return atomic.LoadInt32(&d.synchronising) > 0
+ return atomic.LoadInt32(&d.synchronising) > 0 || atomic.LoadInt32(&d.processing) > 0
}
// RegisterPeer injects a new download peer into the set of block source to be
@@ -233,10 +248,10 @@ func (d *Downloader) UnregisterPeer(id string) error {
// Synchronise tries to sync up our local block chain with a remote peer, both
// adding various sanity checks as well as wrapping it with various log entries.
-func (d *Downloader) Synchronise(id string, head common.Hash, td *big.Int) {
+func (d *Downloader) Synchronise(id string, head common.Hash, td *big.Int, mode SyncMode) {
glog.V(logger.Detail).Infof("Attempting synchronisation: %v, head [%x…], TD %v", id, head[:4], td)
- switch err := d.synchronise(id, head, td); err {
+ switch err := d.synchronise(id, head, td, mode); err {
case nil:
glog.V(logger.Detail).Infof("Synchronisation completed")
@@ -258,7 +273,7 @@ func (d *Downloader) Synchronise(id string, head common.Hash, td *big.Int) {
// synchronise will select the peer and use it for synchronising. If an empty string is given
// it will use the best peer possible and synchronize if it's TD is higher than our own. If any of the
// checks fail an error will be returned. This method is synchronous
-func (d *Downloader) synchronise(id string, hash common.Hash, td *big.Int) error {
+func (d *Downloader) synchronise(id string, hash common.Hash, td *big.Int, mode SyncMode) error {
// Mock out the synchonisation if testing
if d.synchroniseMock != nil {
return d.synchroniseMock(id, hash)
@@ -298,6 +313,11 @@ func (d *Downloader) synchronise(id string, hash common.Hash, td *big.Int) error
d.cancelCh = make(chan struct{})
d.cancelLock.Unlock()
+ // Set the requested sync mode, unless it's forbidden
+ d.mode = mode
+ if d.mode == FastSync && d.noFast {
+ d.mode = FullSync
+ }
// Retrieve the origin peer and initiate the downloading process
p := d.peers.Peer(id)
if p == nil {
@@ -306,13 +326,6 @@ func (d *Downloader) synchronise(id string, hash common.Hash, td *big.Int) error
return d.syncWithPeer(p, hash, td)
}
-/*
-// Has checks if the downloader knows about a particular hash, meaning that its
-// either already downloaded of pending retrieval.
-func (d *Downloader) Has(hash common.Hash) bool {
- return d.queue.Has(hash)
-}
-*/
// syncWithPeer starts a block synchronization based on the hash chain from the
// specified peer and head hash.
func (d *Downloader) syncWithPeer(p *peer, hash common.Hash, td *big.Int) (err error) {
@@ -387,8 +400,28 @@ func (d *Downloader) syncWithPeer(p *peer, hash common.Hash, td *big.Int) (err e
// Initiate the sync using a concurrent header and content retrieval algorithm
pivot := uint64(0)
- if latest > uint64(minFullBlocks) {
- pivot = latest - uint64(minFullBlocks)
+ switch d.mode {
+ case LightSync:
+ pivot = latest
+
+ case FastSync:
+ // Calculate the new fast/slow sync pivot point
+ pivotOffset, err := rand.Int(rand.Reader, big.NewInt(int64(fsPivotInterval)))
+ if err != nil {
+ panic(fmt.Sprintf("Failed to access crypto random source: %v", err))
+ }
+ if latest > uint64(fsMinFullBlocks)+pivotOffset.Uint64() {
+ pivot = latest - uint64(fsMinFullBlocks) - pivotOffset.Uint64()
+ }
+ // If the point is below the origin, move origin back to ensure state download
+ if pivot < origin {
+ if pivot > 0 {
+ origin = pivot - 1
+ } else {
+ origin = 0
+ }
+ }
+ glog.V(logger.Debug).Infof("Fast syncing until pivot block #%d", pivot)
}
d.queue.Prepare(origin+1, d.mode, pivot)
@@ -396,10 +429,10 @@ func (d *Downloader) syncWithPeer(p *peer, hash common.Hash, td *big.Int) (err e
d.syncInitHook(origin, latest)
}
errc := make(chan error, 4)
- go func() { errc <- d.fetchHeaders(p, td, origin+1, latest) }() // Headers are always retrieved
- go func() { errc <- d.fetchBodies(origin + 1) }() // Bodies are retrieved during normal and fast sync
- go func() { errc <- d.fetchReceipts(origin + 1) }() // Receipts are retrieved during fast sync
- go func() { errc <- d.fetchNodeData() }() // Node state data is retrieved during fast sync
+ go func() { errc <- d.fetchHeaders(p, td, origin+1) }() // Headers are always retrieved
+ go func() { errc <- d.fetchBodies(origin + 1) }() // Bodies are retrieved during normal and fast sync
+ go func() { errc <- d.fetchReceipts(origin + 1) }() // Receipts are retrieved during fast sync
+ go func() { errc <- d.fetchNodeData() }() // Node state data is retrieved during fast sync
// If any fetcher fails, cancel the others
var fail error
@@ -844,7 +877,7 @@ func (d *Downloader) fetchBlocks61(from uint64) error {
for _, peer := range idles {
// Short circuit if throttling activated
- if d.queue.ThrottleBlocks() {
+ if d.queue.ShouldThrottleBlocks() {
throttled = true
break
}
@@ -860,8 +893,13 @@ func (d *Downloader) fetchBlocks61(from uint64) error {
}
// Fetch the chunk and make sure any errors return the hashes to the queue
if err := peer.Fetch61(request); err != nil {
- glog.V(logger.Error).Infof("%v: fetch failed, rescheduling", peer)
- d.queue.CancelBlocks(request)
+ // Although we could try and make an attempt to fix this, this error really
+ // means that we've double allocated a fetch task to a peer. If that is the
+ // case, the internal state of the downloader and the queue is very wrong so
+ // better hard crash and note the error instead of silently accumulating into
+ // a much bigger issue.
+ panic(fmt.Sprintf("%v: fetch assignment failed, hard panic", peer))
+ d.queue.CancelBlocks(request) // noop for now
}
}
// Make sure that we have peers available for fetching. If all peers have been tried
@@ -1051,28 +1089,34 @@ func (d *Downloader) findAncestor(p *peer) (uint64, error) {
//
// The queue parameter can be used to switch between queuing headers for block
// body download too, or directly import as pure header chains.
-func (d *Downloader) fetchHeaders(p *peer, td *big.Int, from, latest uint64) error {
+func (d *Downloader) fetchHeaders(p *peer, td *big.Int, from uint64) error {
glog.V(logger.Debug).Infof("%v: downloading headers from #%d", p, from)
defer glog.V(logger.Debug).Infof("%v: header download terminated", p)
+ // Calculate the pivoting point for switching from fast to slow sync
+ pivot := d.queue.FastSyncPivot()
+
// Keep a count of uncertain headers to roll back
rollback := []*types.Header{}
defer func() {
if len(rollback) > 0 {
+ // Flatten the headers and roll them back
hashes := make([]common.Hash, len(rollback))
for i, header := range rollback {
hashes[i] = header.Hash()
}
+ lh, lfb, lb := d.headHeader().Number, d.headFastBlock().Number(), d.headBlock().Number()
d.rollback(hashes)
+ glog.V(logger.Warn).Infof("Rolled back %d headers (LH: %d->%d, FB: %d->%d, LB: %d->%d)",
+ len(hashes), lh, d.headHeader().Number, lfb, d.headFastBlock().Number(), lb, d.headBlock().Number())
+
+ // If we're already past the pivot point, this could be an attack, disable fast sync
+ if rollback[len(rollback)-1].Number.Uint64() > pivot {
+ d.noFast = true
+ }
}
}()
- // Calculate the pivoting point for switching from fast to slow sync
- pivot := uint64(0)
- if d.mode == FastSync && latest > uint64(minFullBlocks) {
- pivot = latest - uint64(minFullBlocks)
- } else if d.mode == LightSync {
- pivot = latest
- }
+
// Create a timeout timer, and the associated hash fetcher
request := time.Now() // time of the last fetch request
timeout := time.NewTimer(0) // timer to dump a non-responsive active peer
@@ -1135,6 +1179,19 @@ func (d *Downloader) fetchHeaders(p *peer, td *big.Int, from, latest uint64) err
if !gotHeaders && td.Cmp(d.getTd(d.headBlock().Hash())) > 0 {
return errStallingPeer
}
+ // If fast or light syncing, ensure promised headers are indeed delivered. This is
+ // needed to detect scenarios where an attacker feeds a bad pivot and then bails out
+ // of delivering the post-pivot blocks that would flag the invalid content.
+ //
+ // This check cannot be executed "as is" for full imports, since blocks may still be
+ // queued for processing when the header download completes. However, as long as the
+ // peer gave us something useful, we're already happy/progressed (above check).
+ if d.mode == FastSync || d.mode == LightSync {
+ if td.Cmp(d.getTd(d.headHeader().Hash())) > 0 {
+ return errStallingPeer
+ }
+ }
+ rollback = nil
return nil
}
gotHeaders = true
@@ -1152,8 +1209,8 @@ func (d *Downloader) fetchHeaders(p *peer, td *big.Int, from, latest uint64) err
}
}
// If we're importing pure headers, verify based on their recentness
- frequency := headerCheckFrequency
- if headers[len(headers)-1].Number.Uint64()+uint64(minCheckedHeaders) > pivot {
+ frequency := fsHeaderCheckFrequency
+ if headers[len(headers)-1].Number.Uint64()+uint64(fsHeaderForceVerify) > pivot {
frequency = 1
}
if n, err := d.insertHeaders(headers, frequency); err != nil {
@@ -1162,11 +1219,8 @@ func (d *Downloader) fetchHeaders(p *peer, td *big.Int, from, latest uint64) err
}
// All verifications passed, store newly found uncertain headers
rollback = append(rollback, unknown...)
- if len(rollback) > minCheckedHeaders {
- rollback = append(rollback[:0], rollback[len(rollback)-minCheckedHeaders:]...)
- }
- if headers[len(headers)-1].Number.Uint64() >= pivot {
- rollback = rollback[:0]
+ if len(rollback) > fsHeaderSafetyNet {
+ rollback = append(rollback[:0], rollback[len(rollback)-fsHeaderSafetyNet:]...)
}
}
if d.mode == FullSync || d.mode == FastSync {
@@ -1230,12 +1284,11 @@ func (d *Downloader) fetchBodies(from uint64) error {
expire = func() []string { return d.queue.ExpireBodies(bodyHardTTL) }
fetch = func(p *peer, req *fetchRequest) error { return p.FetchBodies(req) }
capacity = func(p *peer) int { return p.BlockCapacity() }
- getIdles = func() ([]*peer, int) { return d.peers.BodyIdlePeers() }
- setIdle = func(p *peer) { p.SetBlocksIdle() }
+ setIdle = func(p *peer) { p.SetBodiesIdle() }
)
err := d.fetchParts(errCancelBodyFetch, d.bodyCh, deliver, d.bodyWakeCh, expire,
- d.queue.PendingBlocks, d.queue.InFlightBlocks, d.queue.ThrottleBlocks, d.queue.ReserveBodies,
- d.bodyFetchHook, fetch, d.queue.CancelBodies, capacity, getIdles, setIdle, "Body")
+ d.queue.PendingBlocks, d.queue.InFlightBlocks, d.queue.ShouldThrottleBlocks, d.queue.ReserveBodies,
+ d.bodyFetchHook, fetch, d.queue.CancelBodies, capacity, d.peers.BodyIdlePeers, setIdle, "Body")
glog.V(logger.Debug).Infof("Block body download terminated: %v", err)
return err
@@ -1252,13 +1305,13 @@ func (d *Downloader) fetchReceipts(from uint64) error {
pack := packet.(*receiptPack)
return d.queue.DeliverReceipts(pack.peerId, pack.receipts)
}
- expire = func() []string { return d.queue.ExpireReceipts(bodyHardTTL) }
+ expire = func() []string { return d.queue.ExpireReceipts(receiptHardTTL) }
fetch = func(p *peer, req *fetchRequest) error { return p.FetchReceipts(req) }
capacity = func(p *peer) int { return p.ReceiptCapacity() }
setIdle = func(p *peer) { p.SetReceiptsIdle() }
)
err := d.fetchParts(errCancelReceiptFetch, d.receiptCh, deliver, d.receiptWakeCh, expire,
- d.queue.PendingReceipts, d.queue.InFlightReceipts, d.queue.ThrottleReceipts, d.queue.ReserveReceipts,
+ d.queue.PendingReceipts, d.queue.InFlightReceipts, d.queue.ShouldThrottleReceipts, d.queue.ReserveReceipts,
d.receiptFetchHook, fetch, d.queue.CancelReceipts, capacity, d.peers.ReceiptIdlePeers, setIdle, "Receipt")
glog.V(logger.Debug).Infof("Receipt download terminated: %v", err)
@@ -1307,9 +1360,9 @@ func (d *Downloader) fetchNodeData() error {
capacity = func(p *peer) int { return p.NodeDataCapacity() }
setIdle = func(p *peer) { p.SetNodeDataIdle() }
)
- err := d.fetchParts(errCancelReceiptFetch, d.stateCh, deliver, d.stateWakeCh, expire,
+ err := d.fetchParts(errCancelStateFetch, d.stateCh, deliver, d.stateWakeCh, expire,
d.queue.PendingNodeData, d.queue.InFlightNodeData, throttle, reserve, nil, fetch,
- d.queue.CancelNodeData, capacity, d.peers.ReceiptIdlePeers, setIdle, "State")
+ d.queue.CancelNodeData, capacity, d.peers.NodeDataIdlePeers, setIdle, "State")
glog.V(logger.Debug).Infof("Node state data download terminated: %v", err)
return err
@@ -1323,7 +1376,7 @@ func (d *Downloader) fetchParts(errCancel error, deliveryCh chan dataPack, deliv
fetchHook func([]*types.Header), fetch func(*peer, *fetchRequest) error, cancel func(*fetchRequest), capacity func(*peer) int,
idle func() ([]*peer, int), setIdle func(*peer), kind string) error {
- // Create a ticker to detect expired retreival tasks
+ // Create a ticker to detect expired retrieval tasks
ticker := time.NewTicker(100 * time.Millisecond)
defer ticker.Stop()
@@ -1366,11 +1419,6 @@ func (d *Downloader) fetchParts(errCancel error, deliveryCh chan dataPack, deliv
// The hash chain is invalid (blocks are not ordered properly), abort
return err
- case errInvalidBody, errInvalidReceipt:
- // The peer delivered something very bad, drop immediately
- glog.V(logger.Error).Infof("%s: delivered invalid %s, dropping", peer, strings.ToLower(kind))
- d.dropPeer(peer.id)
-
case errNoFetchesPending:
// Peer probably timed out with its delivery but came through
// in the end, demote, but allow to to pull from this peer.
@@ -1475,8 +1523,13 @@ func (d *Downloader) fetchParts(errCancel error, deliveryCh chan dataPack, deliv
fetchHook(request.Headers)
}
if err := fetch(peer, request); err != nil {
- glog.V(logger.Error).Infof("%v: %s fetch failed, rescheduling", peer, strings.ToLower(kind))
- cancel(request)
+ // Although we could try and make an attempt to fix this, this error really
+ // means that we've double allocated a fetch task to a peer. If that is the
+ // case, the internal state of the downloader and the queue is very wrong so
+ // better hard crash and note the error instead of silently accumulating into
+ // a much bigger issue.
+ panic(fmt.Sprintf("%v: %s fetch assignment failed, hard panic", peer, strings.ToLower(kind)))
+ cancel(request) // noop for now
}
running = true
}
@@ -1526,6 +1579,7 @@ func (d *Downloader) process() {
// Repeat the processing as long as there are results to process
for {
// Fetch the next batch of results
+ pivot := d.queue.FastSyncPivot() // Fetch pivot before results to prevent reset race
results := d.queue.TakeResults()
if len(results) == 0 {
return
@@ -1545,7 +1599,6 @@ func (d *Downloader) process() {
}
// Retrieve the a batch of results to import
var (
- headers = make([]*types.Header, 0, maxResultsProcess)
blocks = make([]*types.Block, 0, maxResultsProcess)
receipts = make([]types.Receipts, 0, maxResultsProcess)
)
@@ -1556,11 +1609,9 @@ func (d *Downloader) process() {
blocks = append(blocks, types.NewBlockWithHeader(result.Header).WithBody(result.Transactions, result.Uncles))
case d.mode == FastSync:
blocks = append(blocks, types.NewBlockWithHeader(result.Header).WithBody(result.Transactions, result.Uncles))
- if result.Header.Number.Uint64() <= d.queue.fastSyncPivot {
+ if result.Header.Number.Uint64() <= pivot {
receipts = append(receipts, result.Receipts)
}
- case d.mode == LightSync:
- headers = append(headers, result.Header)
}
}
// Try to process the results, aborting if there's an error
@@ -1569,12 +1620,10 @@ func (d *Downloader) process() {
index int
)
switch {
- case len(headers) > 0:
- index, err = d.insertHeaders(headers, headerCheckFrequency)
-
case len(receipts) > 0:
index, err = d.insertReceipts(blocks, receipts)
- if err == nil && blocks[len(blocks)-1].NumberU64() == d.queue.fastSyncPivot {
+ if err == nil && blocks[len(blocks)-1].NumberU64() == pivot {
+ glog.V(logger.Debug).Infof("Committing block #%d [%x…] as the new head", blocks[len(blocks)-1].Number(), blocks[len(blocks)-1].Hash().Bytes()[:4])
index, err = len(blocks)-1, d.commitHeadBlock(blocks[len(blocks)-1].Hash())
}
default:
diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go
index f01650ebd..ef6f74a6b 100644
--- a/eth/downloader/downloader_test.go
+++ b/eth/downloader/downloader_test.go
@@ -136,7 +136,7 @@ type downloadTester struct {
}
// newTester creates a new downloader test mocker.
-func newTester(mode SyncMode) *downloadTester {
+func newTester() *downloadTester {
tester := &downloadTester{
ownHashes: []common.Hash{genesis.Hash()},
ownHeaders: map[common.Hash]*types.Header{genesis.Hash(): genesis.Header()},
@@ -150,7 +150,7 @@ func newTester(mode SyncMode) *downloadTester {
peerChainTds: make(map[string]map[common.Hash]*big.Int),
}
tester.stateDb, _ = ethdb.NewMemDatabase()
- tester.downloader = New(mode, tester.stateDb, new(event.TypeMux), tester.hasHeader, tester.hasBlock, tester.getHeader,
+ tester.downloader = New(tester.stateDb, new(event.TypeMux), tester.hasHeader, tester.hasBlock, tester.getHeader,
tester.getBlock, tester.headHeader, tester.headBlock, tester.headFastBlock, tester.commitHeadBlock, tester.getTd,
tester.insertHeaders, tester.insertBlocks, tester.insertReceipts, tester.rollback, tester.dropPeer)
@@ -158,7 +158,7 @@ func newTester(mode SyncMode) *downloadTester {
}
// sync starts synchronizing with a remote peer, blocking until it completes.
-func (dl *downloadTester) sync(id string, td *big.Int) error {
+func (dl *downloadTester) sync(id string, td *big.Int, mode SyncMode) error {
dl.lock.RLock()
hash := dl.peerHashes[id][0]
// If no particular TD was requested, load from the peer's blockchain
@@ -170,7 +170,7 @@ func (dl *downloadTester) sync(id string, td *big.Int) error {
}
dl.lock.RUnlock()
- err := dl.downloader.synchronise(id, hash, td)
+ err := dl.downloader.synchronise(id, hash, td, mode)
for {
// If the queue is empty and processing stopped, break
if dl.downloader.queue.Idle() && atomic.LoadInt32(&dl.downloader.processing) == 0 {
@@ -214,7 +214,7 @@ func (dl *downloadTester) headHeader() *types.Header {
defer dl.lock.RUnlock()
for i := len(dl.ownHashes) - 1; i >= 0; i-- {
- if header := dl.getHeader(dl.ownHashes[i]); header != nil {
+ if header := dl.ownHeaders[dl.ownHashes[i]]; header != nil {
return header
}
}
@@ -227,7 +227,7 @@ func (dl *downloadTester) headBlock() *types.Block {
defer dl.lock.RUnlock()
for i := len(dl.ownHashes) - 1; i >= 0; i-- {
- if block := dl.getBlock(dl.ownHashes[i]); block != nil {
+ if block := dl.ownBlocks[dl.ownHashes[i]]; block != nil {
if _, err := dl.stateDb.Get(block.Root().Bytes()); err == nil {
return block
}
@@ -242,7 +242,7 @@ func (dl *downloadTester) headFastBlock() *types.Block {
defer dl.lock.RUnlock()
for i := len(dl.ownHashes) - 1; i >= 0; i-- {
- if block := dl.getBlock(dl.ownHashes[i]); block != nil {
+ if block := dl.ownBlocks[dl.ownHashes[i]]; block != nil {
return block
}
}
@@ -291,7 +291,7 @@ func (dl *downloadTester) insertHeaders(headers []*types.Header, checkFreq int)
}
dl.ownHashes = append(dl.ownHashes, header.Hash())
dl.ownHeaders[header.Hash()] = header
- dl.ownChainTd[header.Hash()] = dl.ownChainTd[header.ParentHash]
+ dl.ownChainTd[header.Hash()] = new(big.Int).Add(dl.ownChainTd[header.ParentHash], header.Difficulty)
}
return len(headers), nil
}
@@ -305,11 +305,13 @@ func (dl *downloadTester) insertBlocks(blocks types.Blocks) (int, error) {
if _, ok := dl.ownBlocks[block.ParentHash()]; !ok {
return i, errors.New("unknown parent")
}
- dl.ownHashes = append(dl.ownHashes, block.Hash())
- dl.ownHeaders[block.Hash()] = block.Header()
+ if _, ok := dl.ownHeaders[block.Hash()]; !ok {
+ dl.ownHashes = append(dl.ownHashes, block.Hash())
+ dl.ownHeaders[block.Hash()] = block.Header()
+ }
dl.ownBlocks[block.Hash()] = block
- dl.stateDb.Put(block.Root().Bytes(), []byte{})
- dl.ownChainTd[block.Hash()] = dl.ownChainTd[block.ParentHash()]
+ dl.stateDb.Put(block.Root().Bytes(), []byte{0x00})
+ dl.ownChainTd[block.Hash()] = new(big.Int).Add(dl.ownChainTd[block.ParentHash()], block.Difficulty())
}
return len(blocks), nil
}
@@ -381,7 +383,19 @@ func (dl *downloadTester) newSlowPeer(id string, version int, hashes []common.Ha
dl.peerReceipts[id] = make(map[common.Hash]types.Receipts)
dl.peerChainTds[id] = make(map[common.Hash]*big.Int)
- for _, hash := range hashes {
+ genesis := hashes[len(hashes)-1]
+ if header := headers[genesis]; header != nil {
+ dl.peerHeaders[id][genesis] = header
+ dl.peerChainTds[id][genesis] = header.Difficulty
+ }
+ if block := blocks[genesis]; block != nil {
+ dl.peerBlocks[id][genesis] = block
+ dl.peerChainTds[id][genesis] = block.Difficulty()
+ }
+
+ for i := len(hashes) - 2; i >= 0; i-- {
+ hash := hashes[i]
+
if header, ok := headers[hash]; ok {
dl.peerHeaders[id][hash] = header
if _, ok := dl.peerHeaders[id][header.ParentHash]; ok {
@@ -627,21 +641,28 @@ func assertOwnChain(t *testing.T, tester *downloadTester, length int) {
// number of items of the various chain components.
func assertOwnForkedChain(t *testing.T, tester *downloadTester, common int, lengths []int) {
// Initialize the counters for the first fork
- headers, blocks, receipts := lengths[0], lengths[0], lengths[0]-minFullBlocks
- if receipts < 0 {
- receipts = 1
+ headers, blocks := lengths[0], lengths[0]
+
+ minReceipts, maxReceipts := lengths[0]-fsMinFullBlocks-fsPivotInterval, lengths[0]-fsMinFullBlocks
+ if minReceipts < 0 {
+ minReceipts = 1
+ }
+ if maxReceipts < 0 {
+ maxReceipts = 1
}
// Update the counters for each subsequent fork
for _, length := range lengths[1:] {
headers += length - common
blocks += length - common
- receipts += length - common - minFullBlocks
+
+ minReceipts += length - common - fsMinFullBlocks - fsPivotInterval
+ maxReceipts += length - common - fsMinFullBlocks
}
switch tester.downloader.mode {
case FullSync:
- receipts = 1
+ minReceipts, maxReceipts = 1, 1
case LightSync:
- blocks, receipts = 1, 1
+ blocks, minReceipts, maxReceipts = 1, 1, 1
}
if hs := len(tester.ownHeaders); hs != headers {
t.Fatalf("synchronised headers mismatch: have %v, want %v", hs, headers)
@@ -649,14 +670,20 @@ func assertOwnForkedChain(t *testing.T, tester *downloadTester, common int, leng
if bs := len(tester.ownBlocks); bs != blocks {
t.Fatalf("synchronised blocks mismatch: have %v, want %v", bs, blocks)
}
- if rs := len(tester.ownReceipts); rs != receipts {
- t.Fatalf("synchronised receipts mismatch: have %v, want %v", rs, receipts)
+ if rs := len(tester.ownReceipts); rs < minReceipts || rs > maxReceipts {
+ t.Fatalf("synchronised receipts mismatch: have %v, want between [%v, %v]", rs, minReceipts, maxReceipts)
}
// Verify the state trie too for fast syncs
if tester.downloader.mode == FastSync {
- if index := lengths[len(lengths)-1] - minFullBlocks - 1; index > 0 {
- if statedb := state.New(tester.ownHeaders[tester.ownHashes[index]].Root, tester.stateDb); statedb == nil {
- t.Fatalf("state reconstruction failed")
+ index := 0
+ if pivot := int(tester.downloader.queue.fastSyncPivot); pivot < common {
+ index = pivot
+ } else {
+ index = len(tester.ownHashes) - lengths[len(lengths)-1] + int(tester.downloader.queue.fastSyncPivot)
+ }
+ if index > 0 {
+ if statedb, err := state.New(tester.ownHeaders[tester.ownHashes[index]].Root, tester.stateDb); statedb == nil || err != nil {
+ t.Fatalf("state reconstruction failed: %v", err)
}
}
}
@@ -678,11 +705,11 @@ func testCanonicalSynchronisation(t *testing.T, protocol int, mode SyncMode) {
targetBlocks := blockCacheLimit - 15
hashes, headers, blocks, receipts := makeChain(targetBlocks, 0, genesis, nil)
- tester := newTester(mode)
+ tester := newTester()
tester.newPeer("peer", protocol, hashes, headers, blocks, receipts)
// Synchronise with the peer and make sure all relevant data was retrieved
- if err := tester.sync("peer", nil); err != nil {
+ if err := tester.sync("peer", nil, mode); err != nil {
t.Fatalf("failed to synchronise blocks: %v", err)
}
assertOwnChain(t, tester, targetBlocks+1)
@@ -702,7 +729,7 @@ func testThrottling(t *testing.T, protocol int, mode SyncMode) {
targetBlocks := 8 * blockCacheLimit
hashes, headers, blocks, receipts := makeChain(targetBlocks, 0, genesis, nil)
- tester := newTester(mode)
+ tester := newTester()
tester.newPeer("peer", protocol, hashes, headers, blocks, receipts)
// Wrap the importer to allow stepping
@@ -714,7 +741,7 @@ func testThrottling(t *testing.T, protocol int, mode SyncMode) {
// Start a synchronisation concurrently
errc := make(chan error)
go func() {
- errc <- tester.sync("peer", nil)
+ errc <- tester.sync("peer", nil, mode)
}()
// Iteratively take some blocks, always checking the retrieval count
for {
@@ -726,10 +753,11 @@ func testThrottling(t *testing.T, protocol int, mode SyncMode) {
break
}
// Wait a bit for sync to throttle itself
- var cached int
+ var cached, frozen int
for start := time.Now(); time.Since(start) < time.Second; {
time.Sleep(25 * time.Millisecond)
+ tester.lock.RLock()
tester.downloader.queue.lock.RLock()
cached = len(tester.downloader.queue.blockDonePool)
if mode == FastSync {
@@ -739,16 +767,23 @@ func testThrottling(t *testing.T, protocol int, mode SyncMode) {
}
}
}
+ frozen = int(atomic.LoadUint32(&blocked))
+ retrieved = len(tester.ownBlocks)
tester.downloader.queue.lock.RUnlock()
+ tester.lock.RUnlock()
- if cached == blockCacheLimit || len(tester.ownBlocks)+cached+int(atomic.LoadUint32(&blocked)) == targetBlocks+1 {
+ if cached == blockCacheLimit || retrieved+cached+frozen == targetBlocks+1 {
break
}
}
// Make sure we filled up the cache, then exhaust it
time.Sleep(25 * time.Millisecond) // give it a chance to screw up
- if cached != blockCacheLimit && len(tester.ownBlocks)+cached+int(atomic.LoadUint32(&blocked)) != targetBlocks+1 {
- t.Fatalf("block count mismatch: have %v, want %v (owned %v, target %v)", cached, blockCacheLimit, len(tester.ownBlocks), targetBlocks+1)
+
+ tester.lock.RLock()
+ retrieved = len(tester.ownBlocks)
+ tester.lock.RUnlock()
+ if cached != blockCacheLimit && retrieved+cached+frozen != targetBlocks+1 {
+ t.Fatalf("block count mismatch: have %v, want %v (owned %v, blocked %v, target %v)", cached, blockCacheLimit, retrieved, frozen, targetBlocks+1)
}
// Permit the blocked blocks to import
if atomic.LoadUint32(&blocked) > 0 {
@@ -779,18 +814,18 @@ func testForkedSynchronisation(t *testing.T, protocol int, mode SyncMode) {
common, fork := MaxHashFetch, 2*MaxHashFetch
hashesA, hashesB, headersA, headersB, blocksA, blocksB, receiptsA, receiptsB := makeChainFork(common+fork, fork, genesis, nil)
- tester := newTester(mode)
+ tester := newTester()
tester.newPeer("fork A", protocol, hashesA, headersA, blocksA, receiptsA)
tester.newPeer("fork B", protocol, hashesB, headersB, blocksB, receiptsB)
// Synchronise with the peer and make sure all blocks were retrieved
- if err := tester.sync("fork A", nil); err != nil {
+ if err := tester.sync("fork A", nil, mode); err != nil {
t.Fatalf("failed to synchronise blocks: %v", err)
}
assertOwnChain(t, tester, common+fork+1)
// Synchronise with the second peer and make sure that fork is pulled too
- if err := tester.sync("fork B", nil); err != nil {
+ if err := tester.sync("fork B", nil, mode); err != nil {
t.Fatalf("failed to synchronise blocks: %v", err)
}
assertOwnForkedChain(t, tester, common+1, []int{common + fork + 1, common + fork + 1})
@@ -798,7 +833,7 @@ func testForkedSynchronisation(t *testing.T, protocol int, mode SyncMode) {
// Tests that an inactive downloader will not accept incoming hashes and blocks.
func TestInactiveDownloader61(t *testing.T) {
- tester := newTester(FullSync)
+ tester := newTester()
// Check that neither hashes nor blocks are accepted
if err := tester.downloader.DeliverHashes("bad peer", []common.Hash{}); err != errNoSyncActive {
@@ -812,7 +847,7 @@ func TestInactiveDownloader61(t *testing.T) {
// Tests that an inactive downloader will not accept incoming block headers and
// bodies.
func TestInactiveDownloader62(t *testing.T) {
- tester := newTester(FullSync)
+ tester := newTester()
// Check that neither block headers nor bodies are accepted
if err := tester.downloader.DeliverHeaders("bad peer", []*types.Header{}); err != errNoSyncActive {
@@ -826,7 +861,7 @@ func TestInactiveDownloader62(t *testing.T) {
// Tests that an inactive downloader will not accept incoming block headers,
// bodies and receipts.
func TestInactiveDownloader63(t *testing.T) {
- tester := newTester(FullSync)
+ tester := newTester()
// Check that neither block headers nor bodies are accepted
if err := tester.downloader.DeliverHeaders("bad peer", []*types.Header{}); err != errNoSyncActive {
@@ -860,7 +895,7 @@ func testCancel(t *testing.T, protocol int, mode SyncMode) {
}
hashes, headers, blocks, receipts := makeChain(targetBlocks, 0, genesis, nil)
- tester := newTester(mode)
+ tester := newTester()
tester.newPeer("peer", protocol, hashes, headers, blocks, receipts)
// Make sure canceling works with a pristine downloader
@@ -869,7 +904,7 @@ func testCancel(t *testing.T, protocol int, mode SyncMode) {
t.Errorf("download queue not idle")
}
// Synchronise with the peer, but cancel afterwards
- if err := tester.sync("peer", nil); err != nil {
+ if err := tester.sync("peer", nil, mode); err != nil {
t.Fatalf("failed to synchronise blocks: %v", err)
}
tester.downloader.cancel()
@@ -893,12 +928,12 @@ func testMultiSynchronisation(t *testing.T, protocol int, mode SyncMode) {
targetBlocks := targetPeers*blockCacheLimit - 15
hashes, headers, blocks, receipts := makeChain(targetBlocks, 0, genesis, nil)
- tester := newTester(mode)
+ tester := newTester()
for i := 0; i < targetPeers; i++ {
id := fmt.Sprintf("peer #%d", i)
tester.newPeer(id, protocol, hashes[i*blockCacheLimit:], headers, blocks, receipts)
}
- if err := tester.sync("peer #0", nil); err != nil {
+ if err := tester.sync("peer #0", nil, mode); err != nil {
t.Fatalf("failed to synchronise blocks: %v", err)
}
assertOwnChain(t, tester, targetBlocks+1)
@@ -920,14 +955,14 @@ func testMultiProtoSync(t *testing.T, protocol int, mode SyncMode) {
hashes, headers, blocks, receipts := makeChain(targetBlocks, 0, genesis, nil)
// Create peers of every type
- tester := newTester(mode)
- tester.newPeer("peer 61", 61, hashes, headers, blocks, receipts)
- tester.newPeer("peer 62", 62, hashes, headers, blocks, receipts)
+ tester := newTester()
+ tester.newPeer("peer 61", 61, hashes, nil, blocks, nil)
+ tester.newPeer("peer 62", 62, hashes, headers, blocks, nil)
tester.newPeer("peer 63", 63, hashes, headers, blocks, receipts)
tester.newPeer("peer 64", 64, hashes, headers, blocks, receipts)
- // Synchronise with the requestd peer and make sure all blocks were retrieved
- if err := tester.sync(fmt.Sprintf("peer %d", protocol), nil); err != nil {
+ // Synchronise with the requested peer and make sure all blocks were retrieved
+ if err := tester.sync(fmt.Sprintf("peer %d", protocol), nil, mode); err != nil {
t.Fatalf("failed to synchronise blocks: %v", err)
}
assertOwnChain(t, tester, targetBlocks+1)
@@ -955,7 +990,7 @@ func testEmptyShortCircuit(t *testing.T, protocol int, mode SyncMode) {
targetBlocks := 2*blockCacheLimit - 15
hashes, headers, blocks, receipts := makeChain(targetBlocks, 0, genesis, nil)
- tester := newTester(mode)
+ tester := newTester()
tester.newPeer("peer", protocol, hashes, headers, blocks, receipts)
// Instrument the downloader to signal body requests
@@ -967,7 +1002,7 @@ func testEmptyShortCircuit(t *testing.T, protocol int, mode SyncMode) {
atomic.AddInt32(&receiptsHave, int32(len(headers)))
}
// Synchronise with the peer and make sure all blocks were retrieved
- if err := tester.sync("peer", nil); err != nil {
+ if err := tester.sync("peer", nil, mode); err != nil {
t.Fatalf("failed to synchronise blocks: %v", err)
}
assertOwnChain(t, tester, targetBlocks+1)
@@ -980,7 +1015,7 @@ func testEmptyShortCircuit(t *testing.T, protocol int, mode SyncMode) {
}
}
for hash, receipt := range receipts {
- if mode == FastSync && len(receipt) > 0 && headers[hash].Number.Uint64() <= uint64(targetBlocks-minFullBlocks) {
+ if mode == FastSync && len(receipt) > 0 && headers[hash].Number.Uint64() <= tester.downloader.queue.fastSyncPivot {
receiptsNeeded++
}
}
@@ -1006,19 +1041,19 @@ func testMissingHeaderAttack(t *testing.T, protocol int, mode SyncMode) {
targetBlocks := blockCacheLimit - 15
hashes, headers, blocks, receipts := makeChain(targetBlocks, 0, genesis, nil)
- tester := newTester(mode)
+ tester := newTester()
// Attempt a full sync with an attacker feeding gapped headers
tester.newPeer("attack", protocol, hashes, headers, blocks, receipts)
missing := targetBlocks / 2
delete(tester.peerHeaders["attack"], hashes[missing])
- if err := tester.sync("attack", nil); err == nil {
+ if err := tester.sync("attack", nil, mode); err == nil {
t.Fatalf("succeeded attacker synchronisation")
}
// Synchronise with the valid peer and make sure sync succeeds
tester.newPeer("valid", protocol, hashes, headers, blocks, receipts)
- if err := tester.sync("valid", nil); err != nil {
+ if err := tester.sync("valid", nil, mode); err != nil {
t.Fatalf("failed to synchronise blocks: %v", err)
}
assertOwnChain(t, tester, targetBlocks+1)
@@ -1038,7 +1073,7 @@ func testShiftedHeaderAttack(t *testing.T, protocol int, mode SyncMode) {
targetBlocks := blockCacheLimit - 15
hashes, headers, blocks, receipts := makeChain(targetBlocks, 0, genesis, nil)
- tester := newTester(mode)
+ tester := newTester()
// Attempt a full sync with an attacker feeding shifted headers
tester.newPeer("attack", protocol, hashes, headers, blocks, receipts)
@@ -1046,12 +1081,12 @@ func testShiftedHeaderAttack(t *testing.T, protocol int, mode SyncMode) {
delete(tester.peerBlocks["attack"], hashes[len(hashes)-2])
delete(tester.peerReceipts["attack"], hashes[len(hashes)-2])
- if err := tester.sync("attack", nil); err == nil {
+ if err := tester.sync("attack", nil, mode); err == nil {
t.Fatalf("succeeded attacker synchronisation")
}
// Synchronise with the valid peer and make sure sync succeeds
tester.newPeer("valid", protocol, hashes, headers, blocks, receipts)
- if err := tester.sync("valid", nil); err != nil {
+ if err := tester.sync("valid", nil, mode); err != nil {
t.Fatalf("failed to synchronise blocks: %v", err)
}
assertOwnChain(t, tester, targetBlocks+1)
@@ -1064,92 +1099,81 @@ func TestInvalidHeaderRollback64Light(t *testing.T) { testInvalidHeaderRollback(
func testInvalidHeaderRollback(t *testing.T, protocol int, mode SyncMode) {
// Create a small enough block chain to download
- targetBlocks := 3*minCheckedHeaders + minFullBlocks
+ targetBlocks := 3*fsHeaderSafetyNet + fsMinFullBlocks
hashes, headers, blocks, receipts := makeChain(targetBlocks, 0, genesis, nil)
- tester := newTester(mode)
+ tester := newTester()
- // Attempt to sync with an attacker that feeds junk during the fast sync phase
+ // Attempt to sync with an attacker that feeds junk during the fast sync phase.
+ // This should result in the last fsHeaderSafetyNet headers being rolled back.
tester.newPeer("fast-attack", protocol, hashes, headers, blocks, receipts)
- missing := minCheckedHeaders + MaxHeaderFetch + 1
+ missing := fsHeaderSafetyNet + MaxHeaderFetch + 1
delete(tester.peerHeaders["fast-attack"], hashes[len(hashes)-missing])
- if err := tester.sync("fast-attack", nil); err == nil {
+ if err := tester.sync("fast-attack", nil, mode); err == nil {
t.Fatalf("succeeded fast attacker synchronisation")
}
if head := tester.headHeader().Number.Int64(); int(head) > MaxHeaderFetch {
- t.Fatalf("rollback head mismatch: have %v, want at most %v", head, MaxHeaderFetch)
+ t.Errorf("rollback head mismatch: have %v, want at most %v", head, MaxHeaderFetch)
}
- // Attempt to sync with an attacker that feeds junk during the block import phase
+ // Attempt to sync with an attacker that feeds junk during the block import phase.
+ // This should result in both the last fsHeaderSafetyNet number of headers being
+ // rolled back, and also the pivot point being reverted to a non-block status.
tester.newPeer("block-attack", protocol, hashes, headers, blocks, receipts)
- missing = 3*minCheckedHeaders + MaxHeaderFetch + 1
+ missing = 3*fsHeaderSafetyNet + MaxHeaderFetch + 1
delete(tester.peerHeaders["block-attack"], hashes[len(hashes)-missing])
- if err := tester.sync("block-attack", nil); err == nil {
+ if err := tester.sync("block-attack", nil, mode); err == nil {
t.Fatalf("succeeded block attacker synchronisation")
}
+ if head := tester.headHeader().Number.Int64(); int(head) > 2*fsHeaderSafetyNet+MaxHeaderFetch {
+ t.Errorf("rollback head mismatch: have %v, want at most %v", head, 2*fsHeaderSafetyNet+MaxHeaderFetch)
+ }
if mode == FastSync {
- // Fast sync should not discard anything below the verified pivot point
- if head := tester.headHeader().Number.Int64(); int(head) < 3*minCheckedHeaders {
- t.Fatalf("rollback head mismatch: have %v, want at least %v", head, 3*minCheckedHeaders)
+ if head := tester.headBlock().NumberU64(); head != 0 {
+ t.Errorf("fast sync pivot block #%d not rolled back", head)
}
- } else if mode == LightSync {
- // Light sync should still discard data as before
- if head := tester.headHeader().Number.Int64(); int(head) > 3*minCheckedHeaders {
- t.Fatalf("rollback head mismatch: have %v, want at most %v", head, 3*minCheckedHeaders)
- }
- }
- // Synchronise with the valid peer and make sure sync succeeds
- tester.newPeer("valid", protocol, hashes, headers, blocks, receipts)
- if err := tester.sync("valid", nil); err != nil {
- t.Fatalf("failed to synchronise blocks: %v", err)
}
- assertOwnChain(t, tester, targetBlocks+1)
-}
+ // Attempt to sync with an attacker that withholds promised blocks after the
+ // fast sync pivot point. This could be a trial to leave the node with a bad
+ // but already imported pivot block.
+ tester.newPeer("withhold-attack", protocol, hashes, headers, blocks, receipts)
+ missing = 3*fsHeaderSafetyNet + MaxHeaderFetch + 1
-// Tests that if a peer sends an invalid block piece (body or receipt) for a
-// requested block, it gets dropped immediately by the downloader.
-func TestInvalidContentAttack62(t *testing.T) { testInvalidContentAttack(t, 62, FullSync) }
-func TestInvalidContentAttack63Full(t *testing.T) { testInvalidContentAttack(t, 63, FullSync) }
-func TestInvalidContentAttack63Fast(t *testing.T) { testInvalidContentAttack(t, 63, FastSync) }
-func TestInvalidContentAttack64Full(t *testing.T) { testInvalidContentAttack(t, 64, FullSync) }
-func TestInvalidContentAttack64Fast(t *testing.T) { testInvalidContentAttack(t, 64, FastSync) }
-func TestInvalidContentAttack64Light(t *testing.T) { testInvalidContentAttack(t, 64, LightSync) }
-
-func testInvalidContentAttack(t *testing.T, protocol int, mode SyncMode) {
- // Create two peers, one feeding invalid block bodies
- targetBlocks := 4*blockCacheLimit - 15
- hashes, headers, validBlocks, validReceipts := makeChain(targetBlocks, 0, genesis, nil)
-
- invalidBlocks := make(map[common.Hash]*types.Block)
- for hash, block := range validBlocks {
- invalidBlocks[hash] = types.NewBlockWithHeader(block.Header())
- }
- invalidReceipts := make(map[common.Hash]types.Receipts)
- for hash, _ := range validReceipts {
- invalidReceipts[hash] = types.Receipts{&types.Receipt{}}
+ tester.downloader.noFast = false
+ tester.downloader.syncInitHook = func(uint64, uint64) {
+ for i := missing; i <= len(hashes); i++ {
+ delete(tester.peerHeaders["withhold-attack"], hashes[len(hashes)-i])
+ }
+ tester.downloader.syncInitHook = nil
}
- tester := newTester(mode)
- tester.newPeer("valid", protocol, hashes, headers, validBlocks, validReceipts)
- if mode != LightSync {
- tester.newPeer("body attack", protocol, hashes, headers, invalidBlocks, validReceipts)
+ if err := tester.sync("withhold-attack", nil, mode); err == nil {
+ t.Fatalf("succeeded withholding attacker synchronisation")
+ }
+ if head := tester.headHeader().Number.Int64(); int(head) > 2*fsHeaderSafetyNet+MaxHeaderFetch {
+ t.Errorf("rollback head mismatch: have %v, want at most %v", head, 2*fsHeaderSafetyNet+MaxHeaderFetch)
}
if mode == FastSync {
- tester.newPeer("receipt attack", protocol, hashes, headers, validBlocks, invalidReceipts)
+ if head := tester.headBlock().NumberU64(); head != 0 {
+ t.Errorf("fast sync pivot block #%d not rolled back", head)
+ }
}
- // Synchronise with the valid peer (will pull contents from the attacker too)
- if err := tester.sync("valid", nil); err != nil {
+ // Synchronise with the valid peer and make sure sync succeeds. Since the last
+ // rollback should also disable fast syncing for this process, verify that we
+ // did a fresh full sync. Note, we can't assert anything about the receipts
+ // since we won't purge the database of them, hence we can't use asserOwnChain.
+ tester.newPeer("valid", protocol, hashes, headers, blocks, receipts)
+ if err := tester.sync("valid", nil, mode); err != nil {
t.Fatalf("failed to synchronise blocks: %v", err)
}
- assertOwnChain(t, tester, targetBlocks+1)
-
- // Make sure the attacker was detected and dropped in the mean time
- if _, ok := tester.peerHashes["body attack"]; ok {
- t.Fatalf("block body attacker not detected/dropped")
+ if hs := len(tester.ownHeaders); hs != len(headers) {
+ t.Fatalf("synchronised headers mismatch: have %v, want %v", hs, len(headers))
}
- if _, ok := tester.peerHashes["receipt attack"]; ok {
- t.Fatalf("receipt attacker not detected/dropped")
+ if mode != LightSync {
+ if bs := len(tester.ownBlocks); bs != len(blocks) {
+ t.Fatalf("synchronised blocks mismatch: have %v, want %v", bs, len(blocks))
+ }
}
}
@@ -1164,11 +1188,11 @@ func TestHighTDStarvationAttack64Fast(t *testing.T) { testHighTDStarvationAttac
func TestHighTDStarvationAttack64Light(t *testing.T) { testHighTDStarvationAttack(t, 64, LightSync) }
func testHighTDStarvationAttack(t *testing.T, protocol int, mode SyncMode) {
- tester := newTester(mode)
+ tester := newTester()
hashes, headers, blocks, receipts := makeChain(0, 0, genesis, nil)
tester.newPeer("attack", protocol, []common.Hash{hashes[0]}, headers, blocks, receipts)
- if err := tester.sync("attack", big.NewInt(1000000)); err != errStallingPeer {
+ if err := tester.sync("attack", big.NewInt(1000000), mode); err != errStallingPeer {
t.Fatalf("synchronisation error mismatch: have %v, want %v", err, errStallingPeer)
}
}
@@ -1206,7 +1230,7 @@ func testBlockHeaderAttackerDropping(t *testing.T, protocol int) {
{errCancelBodyFetch, false}, // Synchronisation was canceled, origin may be innocent, don't drop
}
// Run the tests and check disconnection status
- tester := newTester(FullSync)
+ tester := newTester()
for i, tt := range tests {
// Register a new peer and ensure it's presence
id := fmt.Sprintf("test %d", i)
@@ -1219,120 +1243,125 @@ func testBlockHeaderAttackerDropping(t *testing.T, protocol int) {
// Simulate a synchronisation and check the required result
tester.downloader.synchroniseMock = func(string, common.Hash) error { return tt.result }
- tester.downloader.Synchronise(id, genesis.Hash(), big.NewInt(1000))
+ tester.downloader.Synchronise(id, genesis.Hash(), big.NewInt(1000), FullSync)
if _, ok := tester.peerHashes[id]; !ok != tt.drop {
t.Errorf("test %d: peer drop mismatch for %v: have %v, want %v", i, tt.result, !ok, tt.drop)
}
}
}
-// Tests that synchronisation boundaries (origin block number and highest block
-// number) is tracked and updated correctly.
-func TestSyncBoundaries61(t *testing.T) { testSyncBoundaries(t, 61, FullSync) }
-func TestSyncBoundaries62(t *testing.T) { testSyncBoundaries(t, 62, FullSync) }
-func TestSyncBoundaries63Full(t *testing.T) { testSyncBoundaries(t, 63, FullSync) }
-func TestSyncBoundaries63Fast(t *testing.T) { testSyncBoundaries(t, 63, FastSync) }
-func TestSyncBoundaries64Full(t *testing.T) { testSyncBoundaries(t, 64, FullSync) }
-func TestSyncBoundaries64Fast(t *testing.T) { testSyncBoundaries(t, 64, FastSync) }
-func TestSyncBoundaries64Light(t *testing.T) { testSyncBoundaries(t, 64, LightSync) }
-
-func testSyncBoundaries(t *testing.T, protocol int, mode SyncMode) {
+// Tests that synchronisation progress (origin block number, current block number
+// and highest block number) is tracked and updated correctly.
+func TestSyncProgress61(t *testing.T) { testSyncProgress(t, 61, FullSync) }
+func TestSyncProgress62(t *testing.T) { testSyncProgress(t, 62, FullSync) }
+func TestSyncProgress63Full(t *testing.T) { testSyncProgress(t, 63, FullSync) }
+func TestSyncProgress63Fast(t *testing.T) { testSyncProgress(t, 63, FastSync) }
+func TestSyncProgress64Full(t *testing.T) { testSyncProgress(t, 64, FullSync) }
+func TestSyncProgress64Fast(t *testing.T) { testSyncProgress(t, 64, FastSync) }
+func TestSyncProgress64Light(t *testing.T) { testSyncProgress(t, 64, LightSync) }
+
+func testSyncProgress(t *testing.T, protocol int, mode SyncMode) {
// Create a small enough block chain to download
targetBlocks := blockCacheLimit - 15
hashes, headers, blocks, receipts := makeChain(targetBlocks, 0, genesis, nil)
- // Set a sync init hook to catch boundary changes
+ // Set a sync init hook to catch progress changes
starting := make(chan struct{})
progress := make(chan struct{})
- tester := newTester(mode)
+ tester := newTester()
tester.downloader.syncInitHook = func(origin, latest uint64) {
starting <- struct{}{}
<-progress
}
- // Retrieve the sync boundaries and ensure they are zero (pristine sync)
- if origin, latest := tester.downloader.Boundaries(); origin != 0 || latest != 0 {
- t.Fatalf("Pristine boundary mismatch: have %v/%v, want %v/%v", origin, latest, 0, 0)
+ // Retrieve the sync progress and ensure they are zero (pristine sync)
+ if origin, current, latest := tester.downloader.Progress(); origin != 0 || current != 0 || latest != 0 {
+ t.Fatalf("Pristine progress mismatch: have %v/%v/%v, want %v/%v/%v", origin, current, latest, 0, 0, 0)
}
- // Synchronise half the blocks and check initial boundaries
+ // Synchronise half the blocks and check initial progress
tester.newPeer("peer-half", protocol, hashes[targetBlocks/2:], headers, blocks, receipts)
pending := new(sync.WaitGroup)
pending.Add(1)
go func() {
defer pending.Done()
- if err := tester.sync("peer-half", nil); err != nil {
+ if err := tester.sync("peer-half", nil, mode); err != nil {
t.Fatalf("failed to synchronise blocks: %v", err)
}
}()
<-starting
- if origin, latest := tester.downloader.Boundaries(); origin != 0 || latest != uint64(targetBlocks/2+1) {
- t.Fatalf("Initial boundary mismatch: have %v/%v, want %v/%v", origin, latest, 0, targetBlocks/2+1)
+ if origin, current, latest := tester.downloader.Progress(); origin != 0 || current != 0 || latest != uint64(targetBlocks/2+1) {
+ t.Fatalf("Initial progress mismatch: have %v/%v/%v, want %v/%v/%v", origin, current, latest, 0, 0, targetBlocks/2+1)
}
progress <- struct{}{}
pending.Wait()
- // Synchronise all the blocks and check continuation boundaries
+ // Synchronise all the blocks and check continuation progress
tester.newPeer("peer-full", protocol, hashes, headers, blocks, receipts)
pending.Add(1)
go func() {
defer pending.Done()
- if err := tester.sync("peer-full", nil); err != nil {
+ if err := tester.sync("peer-full", nil, mode); err != nil {
t.Fatalf("failed to synchronise blocks: %v", err)
}
}()
<-starting
- if origin, latest := tester.downloader.Boundaries(); origin != uint64(targetBlocks/2+1) || latest != uint64(targetBlocks) {
- t.Fatalf("Completing boundary mismatch: have %v/%v, want %v/%v", origin, latest, targetBlocks/2+1, targetBlocks)
+ if origin, current, latest := tester.downloader.Progress(); origin != uint64(targetBlocks/2+1) || current != uint64(targetBlocks/2+1) || latest != uint64(targetBlocks) {
+ t.Fatalf("Completing progress mismatch: have %v/%v/%v, want %v/%v/%v", origin, current, latest, targetBlocks/2+1, targetBlocks/2+1, targetBlocks)
}
progress <- struct{}{}
pending.Wait()
+
+ // Check final progress after successful sync
+ if origin, current, latest := tester.downloader.Progress(); origin != uint64(targetBlocks/2+1) || current != uint64(targetBlocks) || latest != uint64(targetBlocks) {
+ t.Fatalf("Final progress mismatch: have %v/%v/%v, want %v/%v/%v", origin, current, latest, targetBlocks/2+1, targetBlocks, targetBlocks)
+ }
}
-// Tests that synchronisation boundaries (origin block number and highest block
+// Tests that synchronisation progress (origin block number and highest block
// number) is tracked and updated correctly in case of a fork (or manual head
// revertal).
-func TestForkedSyncBoundaries61(t *testing.T) { testForkedSyncBoundaries(t, 61, FullSync) }
-func TestForkedSyncBoundaries62(t *testing.T) { testForkedSyncBoundaries(t, 62, FullSync) }
-func TestForkedSyncBoundaries63Full(t *testing.T) { testForkedSyncBoundaries(t, 63, FullSync) }
-func TestForkedSyncBoundaries63Fast(t *testing.T) { testForkedSyncBoundaries(t, 63, FastSync) }
-func TestForkedSyncBoundaries64Full(t *testing.T) { testForkedSyncBoundaries(t, 64, FullSync) }
-func TestForkedSyncBoundaries64Fast(t *testing.T) { testForkedSyncBoundaries(t, 64, FastSync) }
-func TestForkedSyncBoundaries64Light(t *testing.T) { testForkedSyncBoundaries(t, 64, LightSync) }
-
-func testForkedSyncBoundaries(t *testing.T, protocol int, mode SyncMode) {
+func TestForkedSyncProgress61(t *testing.T) { testForkedSyncProgress(t, 61, FullSync) }
+func TestForkedSyncProgress62(t *testing.T) { testForkedSyncProgress(t, 62, FullSync) }
+func TestForkedSyncProgress63Full(t *testing.T) { testForkedSyncProgress(t, 63, FullSync) }
+func TestForkedSyncProgress63Fast(t *testing.T) { testForkedSyncProgress(t, 63, FastSync) }
+func TestForkedSyncProgress64Full(t *testing.T) { testForkedSyncProgress(t, 64, FullSync) }
+func TestForkedSyncProgress64Fast(t *testing.T) { testForkedSyncProgress(t, 64, FastSync) }
+func TestForkedSyncProgress64Light(t *testing.T) { testForkedSyncProgress(t, 64, LightSync) }
+
+func testForkedSyncProgress(t *testing.T, protocol int, mode SyncMode) {
// Create a forked chain to simulate origin revertal
common, fork := MaxHashFetch, 2*MaxHashFetch
hashesA, hashesB, headersA, headersB, blocksA, blocksB, receiptsA, receiptsB := makeChainFork(common+fork, fork, genesis, nil)
- // Set a sync init hook to catch boundary changes
+ // Set a sync init hook to catch progress changes
starting := make(chan struct{})
progress := make(chan struct{})
- tester := newTester(mode)
+ tester := newTester()
tester.downloader.syncInitHook = func(origin, latest uint64) {
starting <- struct{}{}
<-progress
}
- // Retrieve the sync boundaries and ensure they are zero (pristine sync)
- if origin, latest := tester.downloader.Boundaries(); origin != 0 || latest != 0 {
- t.Fatalf("Pristine boundary mismatch: have %v/%v, want %v/%v", origin, latest, 0, 0)
+ // Retrieve the sync progress and ensure they are zero (pristine sync)
+ if origin, current, latest := tester.downloader.Progress(); origin != 0 || current != 0 || latest != 0 {
+ t.Fatalf("Pristine progress mismatch: have %v/%v/%v, want %v/%v/%v", origin, current, latest, 0, 0, 0)
}
- // Synchronise with one of the forks and check boundaries
+ // Synchronise with one of the forks and check progress
tester.newPeer("fork A", protocol, hashesA, headersA, blocksA, receiptsA)
pending := new(sync.WaitGroup)
pending.Add(1)
go func() {
defer pending.Done()
- if err := tester.sync("fork A", nil); err != nil {
+ if err := tester.sync("fork A", nil, mode); err != nil {
t.Fatalf("failed to synchronise blocks: %v", err)
}
}()
<-starting
- if origin, latest := tester.downloader.Boundaries(); origin != 0 || latest != uint64(len(hashesA)-1) {
- t.Fatalf("Initial boundary mismatch: have %v/%v, want %v/%v", origin, latest, 0, len(hashesA)-1)
+ if origin, current, latest := tester.downloader.Progress(); origin != 0 || current != 0 || latest != uint64(len(hashesA)-1) {
+ t.Fatalf("Initial progress mismatch: have %v/%v/%v, want %v/%v/%v", origin, current, latest, 0, 0, len(hashesA)-1)
}
progress <- struct{}{}
pending.Wait()
@@ -1340,52 +1369,57 @@ func testForkedSyncBoundaries(t *testing.T, protocol int, mode SyncMode) {
// Simulate a successful sync above the fork
tester.downloader.syncStatsChainOrigin = tester.downloader.syncStatsChainHeight
- // Synchronise with the second fork and check boundary resets
+ // Synchronise with the second fork and check progress resets
tester.newPeer("fork B", protocol, hashesB, headersB, blocksB, receiptsB)
pending.Add(1)
go func() {
defer pending.Done()
- if err := tester.sync("fork B", nil); err != nil {
+ if err := tester.sync("fork B", nil, mode); err != nil {
t.Fatalf("failed to synchronise blocks: %v", err)
}
}()
<-starting
- if origin, latest := tester.downloader.Boundaries(); origin != uint64(common) || latest != uint64(len(hashesB)-1) {
- t.Fatalf("Forking boundary mismatch: have %v/%v, want %v/%v", origin, latest, common, len(hashesB)-1)
+ if origin, current, latest := tester.downloader.Progress(); origin != uint64(common) || current != uint64(len(hashesA)-1) || latest != uint64(len(hashesB)-1) {
+ t.Fatalf("Forking progress mismatch: have %v/%v/%v, want %v/%v/%v", origin, current, latest, common, len(hashesA)-1, len(hashesB)-1)
}
progress <- struct{}{}
pending.Wait()
+
+ // Check final progress after successful sync
+ if origin, current, latest := tester.downloader.Progress(); origin != uint64(common) || current != uint64(len(hashesB)-1) || latest != uint64(len(hashesB)-1) {
+ t.Fatalf("Final progress mismatch: have %v/%v/%v, want %v/%v/%v", origin, current, latest, common, len(hashesB)-1, len(hashesB)-1)
+ }
}
-// Tests that if synchronisation is aborted due to some failure, then the boundary
+// Tests that if synchronisation is aborted due to some failure, then the progress
// origin is not updated in the next sync cycle, as it should be considered the
// continuation of the previous sync and not a new instance.
-func TestFailedSyncBoundaries61(t *testing.T) { testFailedSyncBoundaries(t, 61, FullSync) }
-func TestFailedSyncBoundaries62(t *testing.T) { testFailedSyncBoundaries(t, 62, FullSync) }
-func TestFailedSyncBoundaries63Full(t *testing.T) { testFailedSyncBoundaries(t, 63, FullSync) }
-func TestFailedSyncBoundaries63Fast(t *testing.T) { testFailedSyncBoundaries(t, 63, FastSync) }
-func TestFailedSyncBoundaries64Full(t *testing.T) { testFailedSyncBoundaries(t, 64, FullSync) }
-func TestFailedSyncBoundaries64Fast(t *testing.T) { testFailedSyncBoundaries(t, 64, FastSync) }
-func TestFailedSyncBoundaries64Light(t *testing.T) { testFailedSyncBoundaries(t, 64, LightSync) }
-
-func testFailedSyncBoundaries(t *testing.T, protocol int, mode SyncMode) {
+func TestFailedSyncProgress61(t *testing.T) { testFailedSyncProgress(t, 61, FullSync) }
+func TestFailedSyncProgress62(t *testing.T) { testFailedSyncProgress(t, 62, FullSync) }
+func TestFailedSyncProgress63Full(t *testing.T) { testFailedSyncProgress(t, 63, FullSync) }
+func TestFailedSyncProgress63Fast(t *testing.T) { testFailedSyncProgress(t, 63, FastSync) }
+func TestFailedSyncProgress64Full(t *testing.T) { testFailedSyncProgress(t, 64, FullSync) }
+func TestFailedSyncProgress64Fast(t *testing.T) { testFailedSyncProgress(t, 64, FastSync) }
+func TestFailedSyncProgress64Light(t *testing.T) { testFailedSyncProgress(t, 64, LightSync) }
+
+func testFailedSyncProgress(t *testing.T, protocol int, mode SyncMode) {
// Create a small enough block chain to download
targetBlocks := blockCacheLimit - 15
hashes, headers, blocks, receipts := makeChain(targetBlocks, 0, genesis, nil)
- // Set a sync init hook to catch boundary changes
+ // Set a sync init hook to catch progress changes
starting := make(chan struct{})
progress := make(chan struct{})
- tester := newTester(mode)
+ tester := newTester()
tester.downloader.syncInitHook = func(origin, latest uint64) {
starting <- struct{}{}
<-progress
}
- // Retrieve the sync boundaries and ensure they are zero (pristine sync)
- if origin, latest := tester.downloader.Boundaries(); origin != 0 || latest != 0 {
- t.Fatalf("Pristine boundary mismatch: have %v/%v, want %v/%v", origin, latest, 0, 0)
+ // Retrieve the sync progress and ensure they are zero (pristine sync)
+ if origin, current, latest := tester.downloader.Progress(); origin != 0 || current != 0 || latest != 0 {
+ t.Fatalf("Pristine progress mismatch: have %v/%v/%v, want %v/%v/%v", origin, current, latest, 0, 0, 0)
}
// Attempt a full sync with a faulty peer
tester.newPeer("faulty", protocol, hashes, headers, blocks, receipts)
@@ -1399,62 +1433,67 @@ func testFailedSyncBoundaries(t *testing.T, protocol int, mode SyncMode) {
go func() {
defer pending.Done()
- if err := tester.sync("faulty", nil); err == nil {
+ if err := tester.sync("faulty", nil, mode); err == nil {
t.Fatalf("succeeded faulty synchronisation")
}
}()
<-starting
- if origin, latest := tester.downloader.Boundaries(); origin != 0 || latest != uint64(targetBlocks) {
- t.Fatalf("Initial boundary mismatch: have %v/%v, want %v/%v", origin, latest, 0, targetBlocks)
+ if origin, current, latest := tester.downloader.Progress(); origin != 0 || current != 0 || latest != uint64(targetBlocks) {
+ t.Fatalf("Initial progress mismatch: have %v/%v/%v, want %v/%v/%v", origin, current, latest, 0, 0, targetBlocks)
}
progress <- struct{}{}
pending.Wait()
- // Synchronise with a good peer and check that the boundary origin remind the same after a failure
+ // Synchronise with a good peer and check that the progress origin remind the same after a failure
tester.newPeer("valid", protocol, hashes, headers, blocks, receipts)
pending.Add(1)
go func() {
defer pending.Done()
- if err := tester.sync("valid", nil); err != nil {
+ if err := tester.sync("valid", nil, mode); err != nil {
t.Fatalf("failed to synchronise blocks: %v", err)
}
}()
<-starting
- if origin, latest := tester.downloader.Boundaries(); origin != 0 || latest != uint64(targetBlocks) {
- t.Fatalf("Completing boundary mismatch: have %v/%v, want %v/%v", origin, latest, 0, targetBlocks)
+ if origin, current, latest := tester.downloader.Progress(); origin != 0 || current > uint64(targetBlocks/2) || latest != uint64(targetBlocks) {
+ t.Fatalf("Completing progress mismatch: have %v/%v/%v, want %v/0-%v/%v", origin, current, latest, 0, targetBlocks/2, targetBlocks)
}
progress <- struct{}{}
pending.Wait()
+
+ // Check final progress after successful sync
+ if origin, current, latest := tester.downloader.Progress(); origin > uint64(targetBlocks/2) || current != uint64(targetBlocks) || latest != uint64(targetBlocks) {
+ t.Fatalf("Final progress mismatch: have %v/%v/%v, want 0-%v/%v/%v", origin, current, latest, targetBlocks/2, targetBlocks, targetBlocks)
+ }
}
// Tests that if an attacker fakes a chain height, after the attack is detected,
-// the boundary height is successfully reduced at the next sync invocation.
-func TestFakedSyncBoundaries61(t *testing.T) { testFakedSyncBoundaries(t, 61, FullSync) }
-func TestFakedSyncBoundaries62(t *testing.T) { testFakedSyncBoundaries(t, 62, FullSync) }
-func TestFakedSyncBoundaries63Full(t *testing.T) { testFakedSyncBoundaries(t, 63, FullSync) }
-func TestFakedSyncBoundaries63Fast(t *testing.T) { testFakedSyncBoundaries(t, 63, FastSync) }
-func TestFakedSyncBoundaries64Full(t *testing.T) { testFakedSyncBoundaries(t, 64, FullSync) }
-func TestFakedSyncBoundaries64Fast(t *testing.T) { testFakedSyncBoundaries(t, 64, FastSync) }
-func TestFakedSyncBoundaries64Light(t *testing.T) { testFakedSyncBoundaries(t, 64, LightSync) }
-
-func testFakedSyncBoundaries(t *testing.T, protocol int, mode SyncMode) {
+// the progress height is successfully reduced at the next sync invocation.
+func TestFakedSyncProgress61(t *testing.T) { testFakedSyncProgress(t, 61, FullSync) }
+func TestFakedSyncProgress62(t *testing.T) { testFakedSyncProgress(t, 62, FullSync) }
+func TestFakedSyncProgress63Full(t *testing.T) { testFakedSyncProgress(t, 63, FullSync) }
+func TestFakedSyncProgress63Fast(t *testing.T) { testFakedSyncProgress(t, 63, FastSync) }
+func TestFakedSyncProgress64Full(t *testing.T) { testFakedSyncProgress(t, 64, FullSync) }
+func TestFakedSyncProgress64Fast(t *testing.T) { testFakedSyncProgress(t, 64, FastSync) }
+func TestFakedSyncProgress64Light(t *testing.T) { testFakedSyncProgress(t, 64, LightSync) }
+
+func testFakedSyncProgress(t *testing.T, protocol int, mode SyncMode) {
// Create a small block chain
targetBlocks := blockCacheLimit - 15
hashes, headers, blocks, receipts := makeChain(targetBlocks+3, 0, genesis, nil)
- // Set a sync init hook to catch boundary changes
+ // Set a sync init hook to catch progress changes
starting := make(chan struct{})
progress := make(chan struct{})
- tester := newTester(mode)
+ tester := newTester()
tester.downloader.syncInitHook = func(origin, latest uint64) {
starting <- struct{}{}
<-progress
}
- // Retrieve the sync boundaries and ensure they are zero (pristine sync)
- if origin, latest := tester.downloader.Boundaries(); origin != 0 || latest != 0 {
- t.Fatalf("Pristine boundary mismatch: have %v/%v, want %v/%v", origin, latest, 0, 0)
+ // Retrieve the sync progress and ensure they are zero (pristine sync)
+ if origin, current, latest := tester.downloader.Progress(); origin != 0 || current != 0 || latest != 0 {
+ t.Fatalf("Pristine progress mismatch: have %v/%v/%v, want %v/%v/%v", origin, current, latest, 0, 0, 0)
}
// Create and sync with an attacker that promises a higher chain than available
tester.newPeer("attack", protocol, hashes, headers, blocks, receipts)
@@ -1469,31 +1508,36 @@ func testFakedSyncBoundaries(t *testing.T, protocol int, mode SyncMode) {
go func() {
defer pending.Done()
- if err := tester.sync("attack", nil); err == nil {
+ if err := tester.sync("attack", nil, mode); err == nil {
t.Fatalf("succeeded attacker synchronisation")
}
}()
<-starting
- if origin, latest := tester.downloader.Boundaries(); origin != 0 || latest != uint64(targetBlocks+3) {
- t.Fatalf("Initial boundary mismatch: have %v/%v, want %v/%v", origin, latest, 0, targetBlocks+3)
+ if origin, current, latest := tester.downloader.Progress(); origin != 0 || current != 0 || latest != uint64(targetBlocks+3) {
+ t.Fatalf("Initial progress mismatch: have %v/%v/%v, want %v/%v/%v", origin, current, latest, 0, 0, targetBlocks+3)
}
progress <- struct{}{}
pending.Wait()
- // Synchronise with a good peer and check that the boundary height has been reduced to the true value
+ // Synchronise with a good peer and check that the progress height has been reduced to the true value
tester.newPeer("valid", protocol, hashes[3:], headers, blocks, receipts)
pending.Add(1)
go func() {
defer pending.Done()
- if err := tester.sync("valid", nil); err != nil {
+ if err := tester.sync("valid", nil, mode); err != nil {
t.Fatalf("failed to synchronise blocks: %v", err)
}
}()
<-starting
- if origin, latest := tester.downloader.Boundaries(); origin != 0 || latest != uint64(targetBlocks) {
- t.Fatalf("Initial boundary mismatch: have %v/%v, want %v/%v", origin, latest, 0, targetBlocks)
+ if origin, current, latest := tester.downloader.Progress(); origin != 0 || current > uint64(targetBlocks) || latest != uint64(targetBlocks) {
+ t.Fatalf("Completing progress mismatch: have %v/%v/%v, want %v/0-%v/%v", origin, current, latest, 0, targetBlocks, targetBlocks)
}
progress <- struct{}{}
pending.Wait()
+
+ // Check final progress after successful sync
+ if origin, current, latest := tester.downloader.Progress(); origin > uint64(targetBlocks) || current != uint64(targetBlocks) || latest != uint64(targetBlocks) {
+ t.Fatalf("Final progress mismatch: have %v/%v/%v, want 0-%v/%v/%v", origin, current, latest, targetBlocks, targetBlocks, targetBlocks)
+ }
}
diff --git a/eth/downloader/modes.go b/eth/downloader/modes.go
index 8916dbb79..ec339c074 100644
--- a/eth/downloader/modes.go
+++ b/eth/downloader/modes.go
@@ -20,7 +20,7 @@ package downloader
type SyncMode int
const (
- FullSync SyncMode = iota // Synchronise the entire block-chain history from full blocks
- FastSync // Quikcly download the headers, full sync only at the chain head
+ FullSync SyncMode = iota // Synchronise the entire blockchain history from full blocks
+ FastSync // Quickly download the headers, full sync only at the chain head
LightSync // Download only the headers and terminate afterwards
)
diff --git a/eth/downloader/peer.go b/eth/downloader/peer.go
index 5011d5d46..1f457cb15 100644
--- a/eth/downloader/peer.go
+++ b/eth/downloader/peer.go
@@ -124,6 +124,10 @@ func (p *peer) Reset() {
// Fetch61 sends a block retrieval request to the remote peer.
func (p *peer) Fetch61(request *fetchRequest) error {
+ // Sanity check the protocol version
+ if p.version != 61 {
+ panic(fmt.Sprintf("block fetch [eth/61] requested on eth/%d", p.version))
+ }
// Short circuit if the peer is already fetching
if !atomic.CompareAndSwapInt32(&p.blockIdle, 0, 1) {
return errAlreadyFetching
@@ -142,6 +146,10 @@ func (p *peer) Fetch61(request *fetchRequest) error {
// FetchBodies sends a block body retrieval request to the remote peer.
func (p *peer) FetchBodies(request *fetchRequest) error {
+ // Sanity check the protocol version
+ if p.version < 62 {
+ panic(fmt.Sprintf("body fetch [eth/62+] requested on eth/%d", p.version))
+ }
// Short circuit if the peer is already fetching
if !atomic.CompareAndSwapInt32(&p.blockIdle, 0, 1) {
return errAlreadyFetching
@@ -160,6 +168,10 @@ func (p *peer) FetchBodies(request *fetchRequest) error {
// FetchReceipts sends a receipt retrieval request to the remote peer.
func (p *peer) FetchReceipts(request *fetchRequest) error {
+ // Sanity check the protocol version
+ if p.version < 63 {
+ panic(fmt.Sprintf("body fetch [eth/63+] requested on eth/%d", p.version))
+ }
// Short circuit if the peer is already fetching
if !atomic.CompareAndSwapInt32(&p.receiptIdle, 0, 1) {
return errAlreadyFetching
@@ -178,6 +190,10 @@ func (p *peer) FetchReceipts(request *fetchRequest) error {
// FetchNodeData sends a node state data retrieval request to the remote peer.
func (p *peer) FetchNodeData(request *fetchRequest) error {
+ // Sanity check the protocol version
+ if p.version < 63 {
+ panic(fmt.Sprintf("node data fetch [eth/63+] requested on eth/%d", p.version))
+ }
// Short circuit if the peer is already fetching
if !atomic.CompareAndSwapInt32(&p.stateIdle, 0, 1) {
return errAlreadyFetching
@@ -196,35 +212,35 @@ func (p *peer) FetchNodeData(request *fetchRequest) error {
// SetBlocksIdle sets the peer to idle, allowing it to execute new retrieval requests.
// Its block retrieval allowance will also be updated either up- or downwards,
-// depending on whether the previous fetch completed in time or not.
+// depending on whether the previous fetch completed in time.
func (p *peer) SetBlocksIdle() {
p.setIdle(p.blockStarted, blockSoftTTL, blockHardTTL, MaxBlockFetch, &p.blockCapacity, &p.blockIdle)
}
// SetBodiesIdle sets the peer to idle, allowing it to execute new retrieval requests.
// Its block body retrieval allowance will also be updated either up- or downwards,
-// depending on whether the previous fetch completed in time or not.
+// depending on whether the previous fetch completed in time.
func (p *peer) SetBodiesIdle() {
- p.setIdle(p.blockStarted, bodySoftTTL, bodyHardTTL, MaxBlockFetch, &p.blockCapacity, &p.blockIdle)
+ p.setIdle(p.blockStarted, bodySoftTTL, bodyHardTTL, MaxBodyFetch, &p.blockCapacity, &p.blockIdle)
}
// SetReceiptsIdle sets the peer to idle, allowing it to execute new retrieval requests.
// Its receipt retrieval allowance will also be updated either up- or downwards,
-// depending on whether the previous fetch completed in time or not.
+// depending on whether the previous fetch completed in time.
func (p *peer) SetReceiptsIdle() {
p.setIdle(p.receiptStarted, receiptSoftTTL, receiptHardTTL, MaxReceiptFetch, &p.receiptCapacity, &p.receiptIdle)
}
// SetNodeDataIdle sets the peer to idle, allowing it to execute new retrieval
// requests. Its node data retrieval allowance will also be updated either up- or
-// downwards, depending on whether the previous fetch completed in time or not.
+// downwards, depending on whether the previous fetch completed in time.
func (p *peer) SetNodeDataIdle() {
p.setIdle(p.stateStarted, stateSoftTTL, stateSoftTTL, MaxStateFetch, &p.stateCapacity, &p.stateIdle)
}
// setIdle sets the peer to idle, allowing it to execute new retrieval requests.
// Its data retrieval allowance will also be updated either up- or downwards,
-// depending on whether the previous fetch completed in time or not.
+// depending on whether the previous fetch completed in time.
func (p *peer) setIdle(started time.Time, softTTL, hardTTL time.Duration, maxFetch int, capacity, idle *int32) {
// Update the peer's download allowance based on previous performance
scale := 2.0
diff --git a/eth/downloader/queue.go b/eth/downloader/queue.go
index 17fbb1c7f..56b46e285 100644
--- a/eth/downloader/queue.go
+++ b/eth/downloader/queue.go
@@ -56,9 +56,8 @@ type fetchRequest struct {
Time time.Time // Time when the request was made
}
-// fetchResult is the assembly collecting partial results from potentially more
-// than one fetcher routines, until all outstanding retrievals complete and the
-// result as a whole can be processed.
+// fetchResult is a struct collecting partial results from data fetchers until
+// all outstanding pieces complete and the result as a whole can be processed.
type fetchResult struct {
Pending int // Number of data fetches still pending
@@ -89,7 +88,7 @@ type queue struct {
receiptPendPool map[string]*fetchRequest // [eth/63] Currently pending receipt retrieval operations
receiptDonePool map[common.Hash]struct{} // [eth/63] Set of the completed receipt fetches
- stateTaskIndex int // [eth/63] Counter indexing the added hashes to ensure prioritized retrieval order
+ stateTaskIndex int // [eth/63] Counter indexing the added hashes to ensure prioritised retrieval order
stateTaskPool map[common.Hash]int // [eth/63] Pending node data retrieval tasks, mapping to their priority
stateTaskQueue *prque.Prque // [eth/63] Priority queue of the hashes to fetch the node data for
statePendPool map[string]*fetchRequest // [eth/63] Currently pending node data retrieval operations
@@ -97,10 +96,10 @@ type queue struct {
stateDatabase ethdb.Database // [eth/63] Trie database to populate during state reassembly
stateScheduler *state.StateSync // [eth/63] State trie synchronisation scheduler and integrator
stateProcessors int32 // [eth/63] Number of currently running state processors
- stateSchedLock sync.RWMutex // [eth/63] Lock serializing access to the state scheduler
+ stateSchedLock sync.RWMutex // [eth/63] Lock serialising access to the state scheduler
resultCache []*fetchResult // Downloaded but not yet delivered fetch results
- resultOffset uint64 // Offset of the first cached fetch result in the block-chain
+ resultOffset uint64 // Offset of the first cached fetch result in the block chain
lock sync.RWMutex
}
@@ -131,6 +130,9 @@ func (q *queue) Reset() {
q.lock.Lock()
defer q.lock.Unlock()
+ q.stateSchedLock.Lock()
+ defer q.stateSchedLock.Unlock()
+
q.mode = FullSync
q.fastSyncPivot = 0
@@ -233,9 +235,17 @@ func (q *queue) Idle() bool {
return (queued + pending + cached) == 0
}
-// ThrottleBlocks checks if the download should be throttled (active block (body)
+// FastSyncPivot retrieves the currently used fast sync pivot point.
+func (q *queue) FastSyncPivot() uint64 {
+ q.lock.RLock()
+ defer q.lock.RUnlock()
+
+ return q.fastSyncPivot
+}
+
+// ShouldThrottleBlocks checks if the download should be throttled (active block (body)
// fetches exceed block cache).
-func (q *queue) ThrottleBlocks() bool {
+func (q *queue) ShouldThrottleBlocks() bool {
q.lock.RLock()
defer q.lock.RUnlock()
@@ -248,9 +258,9 @@ func (q *queue) ThrottleBlocks() bool {
return pending >= len(q.resultCache)-len(q.blockDonePool)
}
-// ThrottleReceipts checks if the download should be throttled (active receipt
+// ShouldThrottleReceipts checks if the download should be throttled (active receipt
// fetches exceed block cache).
-func (q *queue) ThrottleReceipts() bool {
+func (q *queue) ShouldThrottleReceipts() bool {
q.lock.RLock()
defer q.lock.RUnlock()
@@ -269,7 +279,7 @@ func (q *queue) Schedule61(hashes []common.Hash, fifo bool) []common.Hash {
q.lock.Lock()
defer q.lock.Unlock()
- // Insert all the hashes prioritized in the arrival order
+ // Insert all the hashes prioritised in the arrival order
inserts := make([]common.Hash, 0, len(hashes))
for _, hash := range hashes {
// Skip anything we already have
@@ -297,10 +307,10 @@ func (q *queue) Schedule(headers []*types.Header, from uint64) []*types.Header {
q.lock.Lock()
defer q.lock.Unlock()
- // Insert all the headers prioritized by the contained block number
+ // Insert all the headers prioritised by the contained block number
inserts := make([]*types.Header, 0, len(headers))
for _, header := range headers {
- // Make sure chain order is honored and preserved throughout
+ // Make sure chain order is honoured and preserved throughout
hash := header.Hash()
if header.Number == nil || header.Number.Uint64() != from {
glog.V(logger.Warn).Infof("Header #%v [%x] broke chain ordering, expected %d", header.Number, hash[:4], from)
@@ -347,19 +357,29 @@ func (q *queue) GetHeadResult() *fetchResult {
q.lock.RLock()
defer q.lock.RUnlock()
+ // If there are no results pending, return nil
if len(q.resultCache) == 0 || q.resultCache[0] == nil {
return nil
}
+ // If the next result is still incomplete, return nil
if q.resultCache[0].Pending > 0 {
return nil
}
+ // If the next result is the fast sync pivot...
if q.mode == FastSync && q.resultCache[0].Header.Number.Uint64() == q.fastSyncPivot {
+ // If the pivot state trie is still being pulled, return nil
if len(q.stateTaskPool) > 0 {
return nil
}
if q.PendingNodeData() > 0 {
return nil
}
+ // If the state is done, but not enough post-pivot headers were verified, stall...
+ for i := 0; i < fsHeaderForceVerify; i++ {
+ if i+1 >= len(q.resultCache) || q.resultCache[i+1] == nil {
+ return nil
+ }
+ }
}
return q.resultCache[0]
}
@@ -372,7 +392,7 @@ func (q *queue) TakeResults() []*fetchResult {
// Accumulate all available results
results := []*fetchResult{}
- for _, result := range q.resultCache {
+ for i, result := range q.resultCache {
// Stop if no more results are ready
if result == nil || result.Pending > 0 {
break
@@ -385,6 +405,16 @@ func (q *queue) TakeResults() []*fetchResult {
if q.PendingNodeData() > 0 {
break
}
+ // Even is state fetch is done, ensure post-pivot headers passed verifications
+ safe := true
+ for j := 0; j < fsHeaderForceVerify; j++ {
+ if i+j+1 >= len(q.resultCache) || q.resultCache[i+j+1] == nil {
+ safe = false
+ }
+ }
+ if !safe {
+ break
+ }
}
// If we've just inserted the fast sync pivot, stop as the following batch needs different insertion
if q.mode == FastSync && result.Header.Number.Uint64() == q.fastSyncPivot+1 && len(results) > 0 {
@@ -411,6 +441,9 @@ func (q *queue) TakeResults() []*fetchResult {
// ReserveBlocks reserves a set of block hashes for the given peer, skipping any
// previously failed download.
func (q *queue) ReserveBlocks(p *peer, count int) *fetchRequest {
+ q.lock.Lock()
+ defer q.lock.Unlock()
+
return q.reserveHashes(p, count, q.hashQueue, nil, q.blockPendPool, len(q.resultCache)-len(q.blockDonePool))
}
@@ -430,17 +463,21 @@ func (q *queue) ReserveNodeData(p *peer, count int) *fetchRequest {
}
}
}
+ q.lock.Lock()
+ defer q.lock.Unlock()
+
return q.reserveHashes(p, count, q.stateTaskQueue, generator, q.statePendPool, count)
}
// reserveHashes reserves a set of hashes for the given peer, skipping previously
// failed ones.
+//
+// Note, this method expects the queue lock to be already held for writing. The
+// reason the lock is not obtained in here is because the parameters already need
+// to access the queue, so they already need a lock anyway.
func (q *queue) reserveHashes(p *peer, count int, taskQueue *prque.Prque, taskGen func(int), pendPool map[string]*fetchRequest, maxPending int) *fetchRequest {
- q.lock.Lock()
- defer q.lock.Unlock()
-
- // Short circuit if the peer's already downloading something (sanity check not
- // to corrupt state)
+ // Short circuit if the peer's already downloading something (sanity check to
+ // not corrupt state)
if _, ok := pendPool[p.id]; ok {
return nil
}
@@ -492,30 +529,37 @@ func (q *queue) reserveHashes(p *peer, count int, taskQueue *prque.Prque, taskGe
// previously failed downloads. Beside the next batch of needed fetches, it also
// returns a flag whether empty blocks were queued requiring processing.
func (q *queue) ReserveBodies(p *peer, count int) (*fetchRequest, bool, error) {
- noop := func(header *types.Header) bool {
+ isNoop := func(header *types.Header) bool {
return header.TxHash == types.EmptyRootHash && header.UncleHash == types.EmptyUncleHash
}
- return q.reserveHeaders(p, count, q.blockTaskPool, q.blockTaskQueue, q.blockPendPool, q.blockDonePool, noop)
+ q.lock.Lock()
+ defer q.lock.Unlock()
+
+ return q.reserveHeaders(p, count, q.blockTaskPool, q.blockTaskQueue, q.blockPendPool, q.blockDonePool, isNoop)
}
// ReserveReceipts reserves a set of receipt fetches for the given peer, skipping
// any previously failed downloads. Beside the next batch of needed fetches, it
// also returns a flag whether empty receipts were queued requiring importing.
func (q *queue) ReserveReceipts(p *peer, count int) (*fetchRequest, bool, error) {
- noop := func(header *types.Header) bool {
+ isNoop := func(header *types.Header) bool {
return header.ReceiptHash == types.EmptyRootHash
}
- return q.reserveHeaders(p, count, q.receiptTaskPool, q.receiptTaskQueue, q.receiptPendPool, q.receiptDonePool, noop)
+ q.lock.Lock()
+ defer q.lock.Unlock()
+
+ return q.reserveHeaders(p, count, q.receiptTaskPool, q.receiptTaskQueue, q.receiptPendPool, q.receiptDonePool, isNoop)
}
// reserveHeaders reserves a set of data download operations for a given peer,
// skipping any previously failed ones. This method is a generic version used
// by the individual special reservation functions.
+//
+// Note, this method expects the queue lock to be already held for writing. The
+// reason the lock is not obtained in here is because the parameters already need
+// to access the queue, so they already need a lock anyway.
func (q *queue) reserveHeaders(p *peer, count int, taskPool map[common.Hash]*types.Header, taskQueue *prque.Prque,
- pendPool map[string]*fetchRequest, donePool map[common.Hash]struct{}, noop func(*types.Header) bool) (*fetchRequest, bool, error) {
- q.lock.Lock()
- defer q.lock.Unlock()
-
+ pendPool map[string]*fetchRequest, donePool map[common.Hash]struct{}, isNoop func(*types.Header) bool) (*fetchRequest, bool, error) {
// Short circuit if the pool has been depleted, or if the peer's already
// downloading something (sanity check not to corrupt state)
if taskQueue.Empty() {
@@ -537,7 +581,7 @@ func (q *queue) reserveHeaders(p *peer, count int, taskPool map[common.Hash]*typ
for proc := 0; proc < space && len(send) < count && !taskQueue.Empty(); proc++ {
header := taskQueue.PopItem().(*types.Header)
- // If we're the first to request this task, initialize the result container
+ // If we're the first to request this task, initialise the result container
index := int(header.Number.Int64() - int64(q.resultOffset))
if index >= len(q.resultCache) || index < 0 {
return nil, false, errInvalidChain
@@ -553,7 +597,7 @@ func (q *queue) reserveHeaders(p *peer, count int, taskPool map[common.Hash]*typ
}
}
// If this fetch task is a noop, skip this fetch operation
- if noop(header) {
+ if isNoop(header) {
donePool[header.Hash()] = struct{}{}
delete(taskPool, header.Hash())
@@ -562,7 +606,7 @@ func (q *queue) reserveHeaders(p *peer, count int, taskPool map[common.Hash]*typ
progress = true
continue
}
- // Otherwise if not a known unknown block, add to the retrieve list
+ // Otherwise unless the peer is known not to have the data, add to the retrieve list
if p.ignored.Has(header.Hash()) {
skip = append(skip, header)
} else {
@@ -655,35 +699,48 @@ func (q *queue) Revoke(peerId string) {
}
// ExpireBlocks checks for in flight requests that exceeded a timeout allowance,
-// canceling them and returning the responsible peers for penalization.
+// canceling them and returning the responsible peers for penalisation.
func (q *queue) ExpireBlocks(timeout time.Duration) []string {
+ q.lock.Lock()
+ defer q.lock.Unlock()
+
return q.expire(timeout, q.blockPendPool, q.hashQueue, blockTimeoutMeter)
}
// ExpireBodies checks for in flight block body requests that exceeded a timeout
-// allowance, canceling them and returning the responsible peers for penalization.
+// allowance, canceling them and returning the responsible peers for penalisation.
func (q *queue) ExpireBodies(timeout time.Duration) []string {
+ q.lock.Lock()
+ defer q.lock.Unlock()
+
return q.expire(timeout, q.blockPendPool, q.blockTaskQueue, bodyTimeoutMeter)
}
// ExpireReceipts checks for in flight receipt requests that exceeded a timeout
-// allowance, canceling them and returning the responsible peers for penalization.
+// allowance, canceling them and returning the responsible peers for penalisation.
func (q *queue) ExpireReceipts(timeout time.Duration) []string {
+ q.lock.Lock()
+ defer q.lock.Unlock()
+
return q.expire(timeout, q.receiptPendPool, q.receiptTaskQueue, receiptTimeoutMeter)
}
// ExpireNodeData checks for in flight node data requests that exceeded a timeout
-// allowance, canceling them and returning the responsible peers for penalization.
+// allowance, canceling them and returning the responsible peers for penalisation.
func (q *queue) ExpireNodeData(timeout time.Duration) []string {
+ q.lock.Lock()
+ defer q.lock.Unlock()
+
return q.expire(timeout, q.statePendPool, q.stateTaskQueue, stateTimeoutMeter)
}
// expire is the generic check that move expired tasks from a pending pool back
// into a task pool, returning all entities caught with expired tasks.
+//
+// Note, this method expects the queue lock to be already held for writing. The
+// reason the lock is not obtained in here is because the parameters already need
+// to access the queue, so they already need a lock anyway.
func (q *queue) expire(timeout time.Duration, pendPool map[string]*fetchRequest, taskQueue *prque.Prque, timeoutMeter metrics.Meter) []string {
- q.lock.Lock()
- defer q.lock.Unlock()
-
// Iterate over the expired requests and return each to the queue
peers := []string{}
for id, request := range pendPool {
@@ -764,7 +821,7 @@ func (q *queue) DeliverBlocks(id string, blocks []*types.Block) error {
case len(errs) == 1 && (errs[0] == errInvalidChain || errs[0] == errInvalidBlock):
return errs[0]
- case len(errs) == len(request.Headers):
+ case len(errs) == len(blocks):
return errStaleDelivery
default:
@@ -774,6 +831,9 @@ func (q *queue) DeliverBlocks(id string, blocks []*types.Block) error {
// DeliverBodies injects a block body retrieval response into the results queue.
func (q *queue) DeliverBodies(id string, txLists [][]*types.Transaction, uncleLists [][]*types.Header) error {
+ q.lock.Lock()
+ defer q.lock.Unlock()
+
reconstruct := func(header *types.Header, index int, result *fetchResult) error {
if types.DeriveSha(types.Transactions(txLists[index])) != header.TxHash || types.CalcUncleHash(uncleLists[index]) != header.UncleHash {
return errInvalidBody
@@ -787,6 +847,9 @@ func (q *queue) DeliverBodies(id string, txLists [][]*types.Transaction, uncleLi
// DeliverReceipts injects a receipt retrieval response into the results queue.
func (q *queue) DeliverReceipts(id string, receiptList [][]*types.Receipt) error {
+ q.lock.Lock()
+ defer q.lock.Unlock()
+
reconstruct := func(header *types.Header, index int, result *fetchResult) error {
if types.DeriveSha(types.Receipts(receiptList[index])) != header.ReceiptHash {
return errInvalidReceipt
@@ -798,11 +861,12 @@ func (q *queue) DeliverReceipts(id string, receiptList [][]*types.Receipt) error
}
// deliver injects a data retrieval response into the results queue.
+//
+// Note, this method expects the queue lock to be already held for writing. The
+// reason the lock is not obtained in here is because the parameters already need
+// to access the queue, so they already need a lock anyway.
func (q *queue) deliver(id string, taskPool map[common.Hash]*types.Header, taskQueue *prque.Prque, pendPool map[string]*fetchRequest,
donePool map[common.Hash]struct{}, reqTimer metrics.Timer, results int, reconstruct func(header *types.Header, index int, result *fetchResult) error) error {
- q.lock.Lock()
- defer q.lock.Unlock()
-
// Short circuit if the data was never requested
request := pendPool[id]
if request == nil {
@@ -818,7 +882,10 @@ func (q *queue) deliver(id string, taskPool map[common.Hash]*types.Header, taskQ
}
}
// Assemble each of the results with their headers and retrieved data parts
- errs := make([]error, 0)
+ var (
+ failure error
+ useful bool
+ )
for i, header := range request.Headers {
// Short circuit assembly if no more fetch results are found
if i >= results {
@@ -827,15 +894,16 @@ func (q *queue) deliver(id string, taskPool map[common.Hash]*types.Header, taskQ
// Reconstruct the next result if contents match up
index := int(header.Number.Int64() - int64(q.resultOffset))
if index >= len(q.resultCache) || index < 0 || q.resultCache[index] == nil {
- errs = []error{errInvalidChain}
+ failure = errInvalidChain
break
}
if err := reconstruct(header, i, q.resultCache[index]); err != nil {
- errs = []error{err}
+ failure = err
break
}
donePool[header.Hash()] = struct{}{}
q.resultCache[index].Pending--
+ useful = true
// Clean up a successful fetch
request.Headers[i] = nil
@@ -847,19 +915,16 @@ func (q *queue) deliver(id string, taskPool map[common.Hash]*types.Header, taskQ
taskQueue.Push(header, -float32(header.Number.Uint64()))
}
}
- // If none of the blocks were good, it's a stale delivery
+ // If none of the data was good, it's a stale delivery
switch {
- case len(errs) == 0:
- return nil
-
- case len(errs) == 1 && (errs[0] == errInvalidChain || errs[0] == errInvalidBody || errs[0] == errInvalidReceipt):
- return errs[0]
+ case failure == nil || failure == errInvalidChain:
+ return failure
- case len(errs) == len(request.Headers):
- return errStaleDelivery
+ case useful:
+ return fmt.Errorf("partial failure: %v", failure)
default:
- return fmt.Errorf("multiple failures: %v", errs)
+ return errStaleDelivery
}
}
@@ -876,7 +941,7 @@ func (q *queue) DeliverNodeData(id string, data [][]byte, callback func(error, i
stateReqTimer.UpdateSince(request.Time)
delete(q.statePendPool, id)
- // If no data was retrieved, mark them as unavailable for the origin peer
+ // If no data was retrieved, mark their hashes as unavailable for the origin peer
if len(data) == 0 {
for hash, _ := range request.Hashes {
request.Peer.ignored.Add(hash)
@@ -955,9 +1020,6 @@ func (q *queue) Prepare(offset uint64, mode SyncMode, pivot uint64) {
if q.resultOffset < offset {
q.resultOffset = offset
}
- q.fastSyncPivot = 0
- if mode == FastSync {
- q.fastSyncPivot = pivot
- }
+ q.fastSyncPivot = pivot
q.mode = mode
}