diff options
author | Jeffrey Wilcke <jeffrey@ethereum.org> | 2016-06-13 21:16:09 +0800 |
---|---|---|
committer | GitHub <noreply@github.com> | 2016-06-13 21:16:09 +0800 |
commit | a38be3eb488a349693a9c9905ab015278281f8db (patch) | |
tree | 34fa99ba38f421d9c7ffd308ed544cd5053df228 /core | |
parent | 73c028c40a4f1336a0ab4b9773be0a9d7719777f (diff) | |
parent | f9917c8c7b6d16daadebd72977e56a8adc0382b0 (diff) | |
download | go-tangerine-a38be3eb488a349693a9c9905ab015278281f8db.tar.gz go-tangerine-a38be3eb488a349693a9c9905ab015278281f8db.tar.zst go-tangerine-a38be3eb488a349693a9c9905ab015278281f8db.zip |
Merge pull request #2455 from zsfelfoldi/chaindb
core: improved chain db performance by using sequential keys
Diffstat (limited to 'core')
-rw-r--r-- | core/bench_test.go | 119 | ||||
-rw-r--r-- | core/block_validator.go | 2 | ||||
-rw-r--r-- | core/blockchain.go | 124 | ||||
-rw-r--r-- | core/blockchain_test.go | 30 | ||||
-rw-r--r-- | core/database_util.go | 179 | ||||
-rw-r--r-- | core/database_util_test.go | 80 | ||||
-rw-r--r-- | core/genesis.go | 6 | ||||
-rw-r--r-- | core/headerchain.go | 97 | ||||
-rw-r--r-- | core/vm_env.go | 2 |
9 files changed, 434 insertions, 205 deletions
diff --git a/core/bench_test.go b/core/bench_test.go index ac5b57bc8..c6029499a 100644 --- a/core/bench_test.go +++ b/core/bench_test.go @@ -176,3 +176,122 @@ func benchInsertChain(b *testing.B, disk bool, gen func(int, *BlockGen)) { b.Fatalf("insert error (block %d): %v\n", i, err) } } + +func BenchmarkChainRead_header_10k(b *testing.B) { + benchReadChain(b, false, 10000) +} +func BenchmarkChainRead_full_10k(b *testing.B) { + benchReadChain(b, true, 10000) +} +func BenchmarkChainRead_header_100k(b *testing.B) { + benchReadChain(b, false, 100000) +} +func BenchmarkChainRead_full_100k(b *testing.B) { + benchReadChain(b, true, 100000) +} +func BenchmarkChainRead_header_500k(b *testing.B) { + benchReadChain(b, false, 500000) +} +func BenchmarkChainRead_full_500k(b *testing.B) { + benchReadChain(b, true, 500000) +} +func BenchmarkChainWrite_header_10k(b *testing.B) { + benchWriteChain(b, false, 10000) +} +func BenchmarkChainWrite_full_10k(b *testing.B) { + benchWriteChain(b, true, 10000) +} +func BenchmarkChainWrite_header_100k(b *testing.B) { + benchWriteChain(b, false, 100000) +} +func BenchmarkChainWrite_full_100k(b *testing.B) { + benchWriteChain(b, true, 100000) +} +func BenchmarkChainWrite_header_500k(b *testing.B) { + benchWriteChain(b, false, 500000) +} +func BenchmarkChainWrite_full_500k(b *testing.B) { + benchWriteChain(b, true, 500000) +} + +// makeChainForBench writes a given number of headers or empty blocks/receipts +// into a database. +func makeChainForBench(db ethdb.Database, full bool, count uint64) { + var hash common.Hash + for n := uint64(0); n < count; n++ { + header := &types.Header{ + Coinbase: common.Address{}, + Number: big.NewInt(int64(n)), + ParentHash: hash, + Difficulty: big.NewInt(1), + UncleHash: types.EmptyUncleHash, + TxHash: types.EmptyRootHash, + ReceiptHash: types.EmptyRootHash, + } + hash = header.Hash() + WriteHeader(db, header) + WriteCanonicalHash(db, hash, n) + WriteTd(db, hash, n, big.NewInt(int64(n+1))) + if full || n == 0 { + block := types.NewBlockWithHeader(header) + WriteBody(db, hash, n, block.Body()) + WriteBlockReceipts(db, hash, n, nil) + } + } +} + +func benchWriteChain(b *testing.B, full bool, count uint64) { + for i := 0; i < b.N; i++ { + dir, err := ioutil.TempDir("", "eth-chain-bench") + if err != nil { + b.Fatalf("cannot create temporary directory: %v", err) + } + db, err := ethdb.NewLDBDatabase(dir, 128, 1024) + if err != nil { + b.Fatalf("error opening database at %v: %v", dir, err) + } + makeChainForBench(db, full, count) + db.Close() + os.RemoveAll(dir) + } +} + +func benchReadChain(b *testing.B, full bool, count uint64) { + dir, err := ioutil.TempDir("", "eth-chain-bench") + if err != nil { + b.Fatalf("cannot create temporary directory: %v", err) + } + defer os.RemoveAll(dir) + + db, err := ethdb.NewLDBDatabase(dir, 128, 1024) + if err != nil { + b.Fatalf("error opening database at %v: %v", dir, err) + } + makeChainForBench(db, full, count) + db.Close() + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + db, err := ethdb.NewLDBDatabase(dir, 128, 1024) + if err != nil { + b.Fatalf("error opening database at %v: %v", dir, err) + } + chain, err := NewBlockChain(db, testChainConfig(), FakePow{}, new(event.TypeMux)) + if err != nil { + b.Fatalf("error creating chain: %v", err) + } + + for n := uint64(0); n < count; n++ { + header := chain.GetHeaderByNumber(n) + if full { + hash := header.Hash() + GetBody(db, hash, n) + GetBlockReceipts(db, hash, n) + } + } + + db.Close() + } +} diff --git a/core/block_validator.go b/core/block_validator.go index 801d2572b..f777b9f23 100644 --- a/core/block_validator.go +++ b/core/block_validator.go @@ -72,7 +72,7 @@ func (v *BlockValidator) ValidateBlock(block *types.Block) error { return &KnownBlockError{block.Number(), block.Hash()} } } - parent := v.bc.GetBlock(block.ParentHash()) + parent := v.bc.GetBlock(block.ParentHash(), block.NumberU64()-1) if parent == nil { return ParentError(block.ParentHash()) } diff --git a/core/blockchain.go b/core/blockchain.go index bd84adfe9..95bada5ee 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -54,9 +54,7 @@ var ( ) const ( - headerCacheLimit = 512 bodyCacheLimit = 256 - tdCacheLimit = 1024 blockCacheLimit = 256 maxFutureBlocks = 256 maxTimeFutureBlocks = 30 @@ -151,7 +149,7 @@ func NewBlockChain(chainDb ethdb.Database, config *ChainConfig, pow pow.PoW, mux } // Check the current state of the block hashes and make sure that we do not have any of the bad blocks in our chain for hash, _ := range BadHashes { - if header := bc.GetHeader(hash); header != nil { + if header := bc.GetHeaderByHash(hash); header != nil { glog.V(logger.Error).Infof("Found bad hash, rewinding chain to block #%d [%x…]", header.Number, header.ParentHash[:4]) bc.SetHead(header.Number.Uint64() - 1) glog.V(logger.Error).Infoln("Chain rewind was successful, resuming normal operation") @@ -175,7 +173,7 @@ func (self *BlockChain) loadLastState() error { // Corrupt or empty database, init from scratch self.Reset() } else { - if block := self.GetBlock(head); block != nil { + if block := self.GetBlockByHash(head); block != nil { // Block found, set as the current head self.currentBlock = block } else { @@ -186,7 +184,7 @@ func (self *BlockChain) loadLastState() error { // Restore the last known head header currentHeader := self.currentBlock.Header() if head := GetHeadHeaderHash(self.chainDb); head != (common.Hash{}) { - if header := self.GetHeader(head); header != nil { + if header := self.GetHeaderByHash(head); header != nil { currentHeader = header } } @@ -194,16 +192,16 @@ func (self *BlockChain) loadLastState() error { // Restore the last known head fast block self.currentFastBlock = self.currentBlock if head := GetHeadFastBlockHash(self.chainDb); head != (common.Hash{}) { - if block := self.GetBlock(head); block != nil { + if block := self.GetBlockByHash(head); block != nil { self.currentFastBlock = block } } // Issue a status log and return - headerTd := self.GetTd(self.hc.CurrentHeader().Hash()) - blockTd := self.GetTd(self.currentBlock.Hash()) - fastTd := self.GetTd(self.currentFastBlock.Hash()) + headerTd := self.GetTd(currentHeader.Hash(), currentHeader.Number.Uint64()) + blockTd := self.GetTd(self.currentBlock.Hash(), self.currentBlock.NumberU64()) + fastTd := self.GetTd(self.currentFastBlock.Hash(), self.currentFastBlock.NumberU64()) - glog.V(logger.Info).Infof("Last header: #%d [%x…] TD=%v", self.hc.CurrentHeader().Number, self.hc.CurrentHeader().Hash().Bytes()[:4], headerTd) + glog.V(logger.Info).Infof("Last header: #%d [%x…] TD=%v", currentHeader.Number, currentHeader.Hash().Bytes()[:4], headerTd) glog.V(logger.Info).Infof("Last block: #%d [%x…] TD=%v", self.currentBlock.Number(), self.currentBlock.Hash().Bytes()[:4], blockTd) glog.V(logger.Info).Infof("Fast block: #%d [%x…] TD=%v", self.currentFastBlock.Number(), self.currentFastBlock.Hash().Bytes()[:4], fastTd) @@ -218,8 +216,8 @@ func (bc *BlockChain) SetHead(head uint64) { bc.mu.Lock() defer bc.mu.Unlock() - delFn := func(hash common.Hash) { - DeleteBody(bc.chainDb, hash) + delFn := func(hash common.Hash, num uint64) { + DeleteBody(bc.chainDb, hash, num) } bc.hc.SetHead(head, delFn) @@ -230,11 +228,12 @@ func (bc *BlockChain) SetHead(head uint64) { bc.futureBlocks.Purge() // Update all computed fields to the new head - if bc.currentBlock != nil && bc.hc.CurrentHeader().Number.Uint64() < bc.currentBlock.NumberU64() { - bc.currentBlock = bc.GetBlock(bc.hc.CurrentHeader().Hash()) + currentHeader := bc.hc.CurrentHeader() + if bc.currentBlock != nil && currentHeader.Number.Uint64() < bc.currentBlock.NumberU64() { + bc.currentBlock = bc.GetBlock(currentHeader.Hash(), currentHeader.Number.Uint64()) } - if bc.currentFastBlock != nil && bc.hc.CurrentHeader().Number.Uint64() < bc.currentFastBlock.NumberU64() { - bc.currentFastBlock = bc.GetBlock(bc.hc.CurrentHeader().Hash()) + if bc.currentFastBlock != nil && currentHeader.Number.Uint64() < bc.currentFastBlock.NumberU64() { + bc.currentFastBlock = bc.GetBlock(currentHeader.Hash(), currentHeader.Number.Uint64()) } if bc.currentBlock == nil { @@ -257,7 +256,7 @@ func (bc *BlockChain) SetHead(head uint64) { // irrelevant what the chain contents were prior. func (self *BlockChain) FastSyncCommitHead(hash common.Hash) error { // Make sure that both the block as well at its state trie exists - block := self.GetBlock(hash) + block := self.GetBlockByHash(hash) if block == nil { return fmt.Errorf("non existent block [%x…]", hash[:4]) } @@ -313,7 +312,7 @@ func (self *BlockChain) Status() (td *big.Int, currentBlock common.Hash, genesis self.mu.RLock() defer self.mu.RUnlock() - return self.GetTd(self.currentBlock.Hash()), self.currentBlock.Hash(), self.genesisBlock.Hash() + return self.GetTd(self.currentBlock.Hash(), self.currentBlock.NumberU64()), self.currentBlock.Hash(), self.genesisBlock.Hash() } // SetProcessor sets the processor required for making state modifications. @@ -367,7 +366,7 @@ func (bc *BlockChain) ResetWithGenesisBlock(genesis *types.Block) { defer bc.mu.Unlock() // Prepare the genesis block and reinitialise the chain - if err := bc.hc.WriteTd(genesis.Hash(), genesis.Difficulty()); err != nil { + if err := bc.hc.WriteTd(genesis.Hash(), genesis.NumberU64(), genesis.Difficulty()); err != nil { glog.Fatalf("failed to write genesis block TD: %v", err) } if err := WriteBlock(bc.chainDb, genesis); err != nil { @@ -457,7 +456,7 @@ func (self *BlockChain) GetBody(hash common.Hash) *types.Body { body := cached.(*types.Body) return body } - body := GetBody(self.chainDb, hash) + body := GetBody(self.chainDb, hash, self.hc.GetBlockNumber(hash)) if body == nil { return nil } @@ -473,7 +472,7 @@ func (self *BlockChain) GetBodyRLP(hash common.Hash) rlp.RawValue { if cached, ok := self.bodyRLPCache.Get(hash); ok { return cached.(rlp.RawValue) } - body := GetBodyRLP(self.chainDb, hash) + body := GetBodyRLP(self.chainDb, hash, self.hc.GetBlockNumber(hash)) if len(body) == 0 { return nil } @@ -485,14 +484,14 @@ func (self *BlockChain) GetBodyRLP(hash common.Hash) rlp.RawValue { // HasBlock checks if a block is fully present in the database or not, caching // it if present. func (bc *BlockChain) HasBlock(hash common.Hash) bool { - return bc.GetBlock(hash) != nil + return bc.GetBlockByHash(hash) != nil } // HasBlockAndState checks if a block and associated state trie is fully present // in the database or not, caching it if present. func (bc *BlockChain) HasBlockAndState(hash common.Hash) bool { // Check first that the block itself is known - block := bc.GetBlock(hash) + block := bc.GetBlockByHash(hash) if block == nil { return false } @@ -501,13 +500,14 @@ func (bc *BlockChain) HasBlockAndState(hash common.Hash) bool { return err == nil } -// GetBlock retrieves a block from the database by hash, caching it if found. -func (self *BlockChain) GetBlock(hash common.Hash) *types.Block { +// GetBlock retrieves a block from the database by hash and number, +// caching it if found. +func (self *BlockChain) GetBlock(hash common.Hash, number uint64) *types.Block { // Short circuit if the block's already in the cache, retrieve otherwise if block, ok := self.blockCache.Get(hash); ok { return block.(*types.Block) } - block := GetBlock(self.chainDb, hash) + block := GetBlock(self.chainDb, hash, number) if block == nil { return nil } @@ -516,6 +516,11 @@ func (self *BlockChain) GetBlock(hash common.Hash) *types.Block { return block } +// GetBlockByHash retrieves a block from the database by hash, caching it if found. +func (self *BlockChain) GetBlockByHash(hash common.Hash) *types.Block { + return self.GetBlock(hash, self.hc.GetBlockNumber(hash)) +} + // GetBlockByNumber retrieves a block from the database by number, caching it // (associated with its hash) if found. func (self *BlockChain) GetBlockByNumber(number uint64) *types.Block { @@ -523,19 +528,21 @@ func (self *BlockChain) GetBlockByNumber(number uint64) *types.Block { if hash == (common.Hash{}) { return nil } - return self.GetBlock(hash) + return self.GetBlock(hash, number) } // [deprecated by eth/62] // GetBlocksFromHash returns the block corresponding to hash and up to n-1 ancestors. func (self *BlockChain) GetBlocksFromHash(hash common.Hash, n int) (blocks []*types.Block) { + number := self.hc.GetBlockNumber(hash) for i := 0; i < n; i++ { - block := self.GetBlock(hash) + block := self.GetBlock(hash, number) if block == nil { break } blocks = append(blocks, block) hash = block.ParentHash() + number-- } return } @@ -546,7 +553,7 @@ func (self *BlockChain) GetUnclesInChain(block *types.Block, length int) []*type uncles := []*types.Header{} for i := 0; block != nil && i < length; i++ { uncles = append(uncles, block.Uncles()...) - block = self.GetBlock(block.ParentHash()) + block = self.GetBlock(block.ParentHash(), block.NumberU64()-1) } return uncles } @@ -596,15 +603,16 @@ func (self *BlockChain) Rollback(chain []common.Hash) { for i := len(chain) - 1; i >= 0; i-- { hash := chain[i] - if self.hc.CurrentHeader().Hash() == hash { - self.hc.SetCurrentHeader(self.GetHeader(self.hc.CurrentHeader().ParentHash)) + currentHeader := self.hc.CurrentHeader() + if currentHeader.Hash() == hash { + self.hc.SetCurrentHeader(self.GetHeader(currentHeader.ParentHash, currentHeader.Number.Uint64()-1)) } if self.currentFastBlock.Hash() == hash { - self.currentFastBlock = self.GetBlock(self.currentFastBlock.ParentHash()) + self.currentFastBlock = self.GetBlock(self.currentFastBlock.ParentHash(), self.currentFastBlock.NumberU64()-1) WriteHeadFastBlockHash(self.chainDb, self.currentFastBlock.Hash()) } if self.currentBlock.Hash() == hash { - self.currentBlock = self.GetBlock(self.currentBlock.ParentHash()) + self.currentBlock = self.GetBlock(self.currentBlock.ParentHash(), self.currentBlock.NumberU64()-1) WriteHeadBlockHash(self.chainDb, self.currentBlock.Hash()) } } @@ -678,13 +686,13 @@ func (self *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain } } // Write all the data out into the database - if err := WriteBody(self.chainDb, block.Hash(), block.Body()); err != nil { + if err := WriteBody(self.chainDb, block.Hash(), block.NumberU64(), block.Body()); err != nil { errs[index] = fmt.Errorf("failed to write block body: %v", err) atomic.AddInt32(&failed, 1) glog.Fatal(errs[index]) return } - if err := WriteBlockReceipts(self.chainDb, block.Hash(), receipts); err != nil { + if err := WriteBlockReceipts(self.chainDb, block.Hash(), block.NumberU64(), receipts); err != nil { errs[index] = fmt.Errorf("failed to write block receipts: %v", err) atomic.AddInt32(&failed, 1) glog.Fatal(errs[index]) @@ -737,7 +745,7 @@ func (self *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain // Update the head fast sync block if better self.mu.Lock() head := blockChain[len(errs)-1] - if self.GetTd(self.currentFastBlock.Hash()).Cmp(self.GetTd(head.Hash())) < 0 { + if self.GetTd(self.currentFastBlock.Hash(), self.currentFastBlock.NumberU64()).Cmp(self.GetTd(head.Hash(), head.NumberU64())) < 0 { if err := WriteHeadFastBlockHash(self.chainDb, head.Hash()); err != nil { glog.Fatalf("failed to update head fast block hash: %v", err) } @@ -759,12 +767,12 @@ func (self *BlockChain) WriteBlock(block *types.Block) (status WriteStatus, err defer self.wg.Done() // Calculate the total difficulty of the block - ptd := self.GetTd(block.ParentHash()) + ptd := self.GetTd(block.ParentHash(), block.NumberU64()-1) if ptd == nil { return NonStatTy, ParentError(block.ParentHash()) } - localTd := self.GetTd(self.currentBlock.Hash()) + localTd := self.GetTd(self.currentBlock.Hash(), self.currentBlock.NumberU64()) externTd := new(big.Int).Add(block.Difficulty(), ptd) // Make sure no inconsistent state is leaked during insertion @@ -788,7 +796,7 @@ func (self *BlockChain) WriteBlock(block *types.Block) (status WriteStatus, err status = SideStatTy } // Irrelevant of the canonical status, write the block itself to the database - if err := self.hc.WriteTd(block.Hash(), externTd); err != nil { + if err := self.hc.WriteTd(block.Hash(), block.NumberU64(), externTd); err != nil { glog.Fatalf("failed to write block total difficulty: %v", err) } if err := WriteBlock(self.chainDb, block); err != nil { @@ -887,7 +895,7 @@ func (self *BlockChain) InsertChain(chain types.Blocks) (int, error) { // Create a new statedb using the parent block and report an // error if it fails. if statedb == nil { - statedb, err = state.New(self.GetBlock(block.ParentHash()).Root(), self.chainDb) + statedb, err = state.New(self.GetBlock(block.ParentHash(), block.NumberU64()-1).Root(), self.chainDb) } else { err = statedb.Reset(chain[i-1].Root()) } @@ -902,7 +910,7 @@ func (self *BlockChain) InsertChain(chain types.Blocks) (int, error) { return i, err } // Validate the state using the default validator - err = self.Validator().ValidateState(block, self.GetBlock(block.ParentHash()), statedb, receipts, usedGas) + err = self.Validator().ValidateState(block, self.GetBlock(block.ParentHash(), block.NumberU64()-1), statedb, receipts, usedGas) if err != nil { reportBlock(block, err) return i, err @@ -916,7 +924,7 @@ func (self *BlockChain) InsertChain(chain types.Blocks) (int, error) { // coalesce logs for later processing coalescedLogs = append(coalescedLogs, logs...) - if err := WriteBlockReceipts(self.chainDb, block.Hash(), receipts); err != nil { + if err := WriteBlockReceipts(self.chainDb, block.Hash(), block.NumberU64(), receipts); err != nil { return i, err } @@ -986,7 +994,7 @@ func (self *BlockChain) reorg(oldBlock, newBlock *types.Block) error { // These logs are later announced as deleted. collectLogs = func(h common.Hash) { // Coalesce logs - receipts := GetBlockReceipts(self.chainDb, h) + receipts := GetBlockReceipts(self.chainDb, h, self.hc.GetBlockNumber(h)) for _, receipt := range receipts { deletedLogs = append(deletedLogs, receipt.Logs...) @@ -998,7 +1006,7 @@ func (self *BlockChain) reorg(oldBlock, newBlock *types.Block) error { // first reduce whoever is higher bound if oldBlock.NumberU64() > newBlock.NumberU64() { // reduce old chain - for ; oldBlock != nil && oldBlock.NumberU64() != newBlock.NumberU64(); oldBlock = self.GetBlock(oldBlock.ParentHash()) { + for ; oldBlock != nil && oldBlock.NumberU64() != newBlock.NumberU64(); oldBlock = self.GetBlock(oldBlock.ParentHash(), oldBlock.NumberU64()-1) { oldChain = append(oldChain, oldBlock) deletedTxs = append(deletedTxs, oldBlock.Transactions()...) @@ -1006,7 +1014,7 @@ func (self *BlockChain) reorg(oldBlock, newBlock *types.Block) error { } } else { // reduce new chain and append new chain blocks for inserting later on - for ; newBlock != nil && newBlock.NumberU64() != oldBlock.NumberU64(); newBlock = self.GetBlock(newBlock.ParentHash()) { + for ; newBlock != nil && newBlock.NumberU64() != oldBlock.NumberU64(); newBlock = self.GetBlock(newBlock.ParentHash(), newBlock.NumberU64()-1) { newChain = append(newChain, newBlock) } } @@ -1029,7 +1037,7 @@ func (self *BlockChain) reorg(oldBlock, newBlock *types.Block) error { deletedTxs = append(deletedTxs, oldBlock.Transactions()...) collectLogs(oldBlock.Hash()) - oldBlock, newBlock = self.GetBlock(oldBlock.ParentHash()), self.GetBlock(newBlock.ParentHash()) + oldBlock, newBlock = self.GetBlock(oldBlock.ParentHash(), oldBlock.NumberU64()-1), self.GetBlock(newBlock.ParentHash(), newBlock.NumberU64()-1) if oldBlock == nil { return fmt.Errorf("Invalid old chain") } @@ -1052,7 +1060,7 @@ func (self *BlockChain) reorg(oldBlock, newBlock *types.Block) error { if err := WriteTransactions(self.chainDb, block); err != nil { return err } - receipts := GetBlockReceipts(self.chainDb, block.Hash()) + receipts := GetBlockReceipts(self.chainDb, block.Hash(), block.NumberU64()) // write receipts if err := WriteReceipts(self.chainDb, receipts); err != nil { return err @@ -1187,15 +1195,27 @@ func (self *BlockChain) CurrentHeader() *types.Header { } // GetTd retrieves a block's total difficulty in the canonical chain from the +// database by hash and number, caching it if found. +func (self *BlockChain) GetTd(hash common.Hash, number uint64) *big.Int { + return self.hc.GetTd(hash, number) +} + +// GetTdByHash retrieves a block's total difficulty in the canonical chain from the // database by hash, caching it if found. -func (self *BlockChain) GetTd(hash common.Hash) *big.Int { - return self.hc.GetTd(hash) +func (self *BlockChain) GetTdByHash(hash common.Hash) *big.Int { + return self.hc.GetTdByHash(hash) +} + +// GetHeader retrieves a block header from the database by hash and number, +// caching it if found. +func (self *BlockChain) GetHeader(hash common.Hash, number uint64) *types.Header { + return self.hc.GetHeader(hash, number) } -// GetHeader retrieves a block header from the database by hash, caching it if +// GetHeaderByHash retrieves a block header from the database by hash, caching it if // found. -func (self *BlockChain) GetHeader(hash common.Hash) *types.Header { - return self.hc.GetHeader(hash) +func (self *BlockChain) GetHeaderByHash(hash common.Hash) *types.Header { + return self.hc.GetHeaderByHash(hash) } // HasHeader checks if a block header is present in the database or not, caching diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 876dd2ba1..a26fe4a1b 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -102,17 +102,17 @@ func testFork(t *testing.T, blockchain *BlockChain, i, n int, full bool, compara var tdPre, tdPost *big.Int if full { - tdPre = blockchain.GetTd(blockchain.CurrentBlock().Hash()) + tdPre = blockchain.GetTdByHash(blockchain.CurrentBlock().Hash()) if err := testBlockChainImport(blockChainB, blockchain); err != nil { t.Fatalf("failed to import forked block chain: %v", err) } - tdPost = blockchain.GetTd(blockChainB[len(blockChainB)-1].Hash()) + tdPost = blockchain.GetTdByHash(blockChainB[len(blockChainB)-1].Hash()) } else { - tdPre = blockchain.GetTd(blockchain.CurrentHeader().Hash()) + tdPre = blockchain.GetTdByHash(blockchain.CurrentHeader().Hash()) if err := testHeaderChainImport(headerChainB, blockchain); err != nil { t.Fatalf("failed to import forked header chain: %v", err) } - tdPost = blockchain.GetTd(headerChainB[len(headerChainB)-1].Hash()) + tdPost = blockchain.GetTdByHash(headerChainB[len(headerChainB)-1].Hash()) } // Compare the total difficulties of the chains comparator(tdPre, tdPost) @@ -137,7 +137,7 @@ func testBlockChainImport(chain types.Blocks, blockchain *BlockChain) error { } return err } - statedb, err := state.New(blockchain.GetBlock(block.ParentHash()).Root(), blockchain.chainDb) + statedb, err := state.New(blockchain.GetBlockByHash(block.ParentHash()).Root(), blockchain.chainDb) if err != nil { return err } @@ -146,13 +146,13 @@ func testBlockChainImport(chain types.Blocks, blockchain *BlockChain) error { reportBlock(block, err) return err } - err = blockchain.Validator().ValidateState(block, blockchain.GetBlock(block.ParentHash()), statedb, receipts, usedGas) + err = blockchain.Validator().ValidateState(block, blockchain.GetBlockByHash(block.ParentHash()), statedb, receipts, usedGas) if err != nil { reportBlock(block, err) return err } blockchain.mu.Lock() - WriteTd(blockchain.chainDb, block.Hash(), new(big.Int).Add(block.Difficulty(), blockchain.GetTd(block.ParentHash()))) + WriteTd(blockchain.chainDb, block.Hash(), block.NumberU64(), new(big.Int).Add(block.Difficulty(), blockchain.GetTdByHash(block.ParentHash()))) WriteBlock(blockchain.chainDb, block) statedb.Commit() blockchain.mu.Unlock() @@ -165,12 +165,12 @@ func testBlockChainImport(chain types.Blocks, blockchain *BlockChain) error { func testHeaderChainImport(chain []*types.Header, blockchain *BlockChain) error { for _, header := range chain { // Try and validate the header - if err := blockchain.Validator().ValidateHeader(header, blockchain.GetHeader(header.ParentHash), false); err != nil { + if err := blockchain.Validator().ValidateHeader(header, blockchain.GetHeaderByHash(header.ParentHash), false); err != nil { return err } // Manually insert the header into the database, but don't reorganise (allows subsequent testing) blockchain.mu.Lock() - WriteTd(blockchain.chainDb, header.Hash(), new(big.Int).Add(header.Difficulty, blockchain.GetTd(header.ParentHash))) + WriteTd(blockchain.chainDb, header.Hash(), header.Number.Uint64(), new(big.Int).Add(header.Difficulty, blockchain.GetTdByHash(header.ParentHash))) WriteHeader(blockchain.chainDb, header) blockchain.mu.Unlock() } @@ -543,11 +543,11 @@ func testReorg(t *testing.T, first, second []int, td int64, full bool) { // Make sure the chain total difficulty is the correct one want := new(big.Int).Add(genesis.Difficulty(), big.NewInt(td)) if full { - if have := bc.GetTd(bc.CurrentBlock().Hash()); have.Cmp(want) != 0 { + if have := bc.GetTdByHash(bc.CurrentBlock().Hash()); have.Cmp(want) != 0 { t.Errorf("total difficulty mismatch: have %v, want %v", have, want) } } else { - if have := bc.GetTd(bc.CurrentHeader().Hash()); have.Cmp(want) != 0 { + if have := bc.GetTdByHash(bc.CurrentHeader().Hash()); have.Cmp(want) != 0 { t.Errorf("total difficulty mismatch: have %v, want %v", have, want) } } @@ -758,20 +758,20 @@ func TestFastVsFullChains(t *testing.T) { for i := 0; i < len(blocks); i++ { num, hash := blocks[i].NumberU64(), blocks[i].Hash() - if ftd, atd := fast.GetTd(hash), archive.GetTd(hash); ftd.Cmp(atd) != 0 { + if ftd, atd := fast.GetTdByHash(hash), archive.GetTdByHash(hash); ftd.Cmp(atd) != 0 { t.Errorf("block #%d [%x]: td mismatch: have %v, want %v", num, hash, ftd, atd) } - if fheader, aheader := fast.GetHeader(hash), archive.GetHeader(hash); fheader.Hash() != aheader.Hash() { + if fheader, aheader := fast.GetHeaderByHash(hash), archive.GetHeaderByHash(hash); fheader.Hash() != aheader.Hash() { t.Errorf("block #%d [%x]: header mismatch: have %v, want %v", num, hash, fheader, aheader) } - if fblock, ablock := fast.GetBlock(hash), archive.GetBlock(hash); fblock.Hash() != ablock.Hash() { + if fblock, ablock := fast.GetBlockByHash(hash), archive.GetBlockByHash(hash); fblock.Hash() != ablock.Hash() { t.Errorf("block #%d [%x]: block mismatch: have %v, want %v", num, hash, fblock, ablock) } else if types.DeriveSha(fblock.Transactions()) != types.DeriveSha(ablock.Transactions()) { t.Errorf("block #%d [%x]: transactions mismatch: have %v, want %v", num, hash, fblock.Transactions(), ablock.Transactions()) } else if types.CalcUncleHash(fblock.Uncles()) != types.CalcUncleHash(ablock.Uncles()) { t.Errorf("block #%d [%x]: uncles mismatch: have %v, want %v", num, hash, fblock.Uncles(), ablock.Uncles()) } - if freceipts, areceipts := GetBlockReceipts(fastDb, hash), GetBlockReceipts(archiveDb, hash); types.DeriveSha(freceipts) != types.DeriveSha(areceipts) { + if freceipts, areceipts := GetBlockReceipts(fastDb, hash, GetBlockNumber(fastDb, hash)), GetBlockReceipts(archiveDb, hash, GetBlockNumber(archiveDb, hash)); types.DeriveSha(freceipts) != types.DeriveSha(areceipts) { t.Errorf("block #%d [%x]: receipts mismatch: have %v, want %v", num, hash, freceipts, areceipts) } } diff --git a/core/database_util.go b/core/database_util.go index 3ba80062c..1529d7369 100644 --- a/core/database_util.go +++ b/core/database_util.go @@ -36,34 +36,72 @@ var ( headBlockKey = []byte("LastBlock") headFastKey = []byte("LastFast") - blockPrefix = []byte("block-") - blockNumPrefix = []byte("block-num-") + headerPrefix = []byte("h") // headerPrefix + num (uint64 big endian) + hash -> header + tdSuffix = []byte("t") // headerPrefix + num (uint64 big endian) + hash + tdSuffix -> td + numSuffix = []byte("n") // headerPrefix + num (uint64 big endian) + numSuffix -> hash + blockHashPrefix = []byte("H") // blockHashPrefix + hash -> num (uint64 big endian) + bodyPrefix = []byte("b") // bodyPrefix + num (uint64 big endian) + hash -> block body + blockReceiptsPrefix = []byte("r") // blockReceiptsPrefix + num (uint64 big endian) + hash -> block receipts - headerSuffix = []byte("-header") - bodySuffix = []byte("-body") - tdSuffix = []byte("-td") - - txMetaSuffix = []byte{0x01} - receiptsPrefix = []byte("receipts-") - blockReceiptsPrefix = []byte("receipts-block-") + txMetaSuffix = []byte{0x01} + receiptsPrefix = []byte("receipts-") mipmapPre = []byte("mipmap-log-bloom-") MIPMapLevels = []uint64{1000000, 500000, 100000, 50000, 1000} - blockHashPrefix = []byte("block-hash-") // [deprecated by the header/block split, remove eventually] - configPrefix = []byte("ethereum-config-") // config prefix for the db + + // used by old (non-sequential keys) db, now only used for conversion + oldBlockPrefix = []byte("block-") + oldHeaderSuffix = []byte("-header") + oldTdSuffix = []byte("-td") // headerPrefix + num (uint64 big endian) + hash + tdSuffix -> td + oldBodySuffix = []byte("-body") + oldBlockNumPrefix = []byte("block-num-") + oldBlockReceiptsPrefix = []byte("receipts-block-") + oldBlockHashPrefix = []byte("block-hash-") // [deprecated by the header/block split, remove eventually] ) +// encodeBlockNumber encodes a block number as big endian uint64 +func encodeBlockNumber(number uint64) []byte { + enc := make([]byte, 8) + binary.BigEndian.PutUint64(enc, number) + return enc +} + // GetCanonicalHash retrieves a hash assigned to a canonical block number. func GetCanonicalHash(db ethdb.Database, number uint64) common.Hash { - data, _ := db.Get(append(blockNumPrefix, big.NewInt(int64(number)).Bytes()...)) + data, _ := db.Get(append(append(headerPrefix, encodeBlockNumber(number)...), numSuffix...)) if len(data) == 0 { - return common.Hash{} + data, _ = db.Get(append(oldBlockNumPrefix, big.NewInt(int64(number)).Bytes()...)) + if len(data) == 0 { + return common.Hash{} + } } return common.BytesToHash(data) } +// missingNumber is returned by GetBlockNumber if no header with the +// given block hash has been stored in the database +const missingNumber = uint64(0xffffffffffffffff) + +// GetBlockNumber returns the block number assigned to a block hash +// if the corresponding header is present in the database +func GetBlockNumber(db ethdb.Database, hash common.Hash) uint64 { + data, _ := db.Get(append(blockHashPrefix, hash.Bytes()...)) + if len(data) != 8 { + data, _ := db.Get(append(append(oldBlockPrefix, hash.Bytes()...), oldHeaderSuffix...)) + if len(data) == 0 { + return missingNumber + } + header := new(types.Header) + if err := rlp.Decode(bytes.NewReader(data), header); err != nil { + glog.Fatalf("failed to decode block header: %v", err) + } + return header.Number.Uint64() + } + return binary.BigEndian.Uint64(data) +} + // GetHeadHeaderHash retrieves the hash of the current canonical head block's // header. The difference between this and GetHeadBlockHash is that whereas the // last block hash is only updated upon a full block import, the last header @@ -100,15 +138,18 @@ func GetHeadFastBlockHash(db ethdb.Database) common.Hash { // GetHeaderRLP retrieves a block header in its raw RLP database encoding, or nil // if the header's not found. -func GetHeaderRLP(db ethdb.Database, hash common.Hash) rlp.RawValue { - data, _ := db.Get(append(append(blockPrefix, hash[:]...), headerSuffix...)) +func GetHeaderRLP(db ethdb.Database, hash common.Hash, number uint64) rlp.RawValue { + data, _ := db.Get(append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...)) + if len(data) == 0 { + data, _ = db.Get(append(append(oldBlockPrefix, hash.Bytes()...), oldHeaderSuffix...)) + } return data } // GetHeader retrieves the block header corresponding to the hash, nil if none // found. -func GetHeader(db ethdb.Database, hash common.Hash) *types.Header { - data := GetHeaderRLP(db, hash) +func GetHeader(db ethdb.Database, hash common.Hash, number uint64) *types.Header { + data := GetHeaderRLP(db, hash, number) if len(data) == 0 { return nil } @@ -121,15 +162,18 @@ func GetHeader(db ethdb.Database, hash common.Hash) *types.Header { } // GetBodyRLP retrieves the block body (transactions and uncles) in RLP encoding. -func GetBodyRLP(db ethdb.Database, hash common.Hash) rlp.RawValue { - data, _ := db.Get(append(append(blockPrefix, hash[:]...), bodySuffix...)) +func GetBodyRLP(db ethdb.Database, hash common.Hash, number uint64) rlp.RawValue { + data, _ := db.Get(append(append(bodyPrefix, encodeBlockNumber(number)...), hash.Bytes()...)) + if len(data) == 0 { + data, _ = db.Get(append(append(oldBlockPrefix, hash.Bytes()...), oldBodySuffix...)) + } return data } // GetBody retrieves the block body (transactons, uncles) corresponding to the // hash, nil if none found. -func GetBody(db ethdb.Database, hash common.Hash) *types.Body { - data := GetBodyRLP(db, hash) +func GetBody(db ethdb.Database, hash common.Hash, number uint64) *types.Body { + data := GetBodyRLP(db, hash, number) if len(data) == 0 { return nil } @@ -143,10 +187,13 @@ func GetBody(db ethdb.Database, hash common.Hash) *types.Body { // GetTd retrieves a block's total difficulty corresponding to the hash, nil if // none found. -func GetTd(db ethdb.Database, hash common.Hash) *big.Int { - data, _ := db.Get(append(append(blockPrefix, hash.Bytes()...), tdSuffix...)) +func GetTd(db ethdb.Database, hash common.Hash, number uint64) *big.Int { + data, _ := db.Get(append(append(append(headerPrefix, encodeBlockNumber(number)...), hash[:]...), tdSuffix...)) if len(data) == 0 { - return nil + data, _ = db.Get(append(append(oldBlockPrefix, hash.Bytes()...), oldTdSuffix...)) + if len(data) == 0 { + return nil + } } td := new(big.Int) if err := rlp.Decode(bytes.NewReader(data), td); err != nil { @@ -158,13 +205,13 @@ func GetTd(db ethdb.Database, hash common.Hash) *big.Int { // GetBlock retrieves an entire block corresponding to the hash, assembling it // back from the stored header and body. -func GetBlock(db ethdb.Database, hash common.Hash) *types.Block { +func GetBlock(db ethdb.Database, hash common.Hash, number uint64) *types.Block { // Retrieve the block header and body contents - header := GetHeader(db, hash) + header := GetHeader(db, hash, number) if header == nil { return nil } - body := GetBody(db, hash) + body := GetBody(db, hash, number) if body == nil { return nil } @@ -174,10 +221,13 @@ func GetBlock(db ethdb.Database, hash common.Hash) *types.Block { // GetBlockReceipts retrieves the receipts generated by the transactions included // in a block given by its hash. -func GetBlockReceipts(db ethdb.Database, hash common.Hash) types.Receipts { - data, _ := db.Get(append(blockReceiptsPrefix, hash[:]...)) +func GetBlockReceipts(db ethdb.Database, hash common.Hash, number uint64) types.Receipts { + data, _ := db.Get(append(append(blockReceiptsPrefix, encodeBlockNumber(number)...), hash[:]...)) if len(data) == 0 { - return nil + data, _ = db.Get(append(oldBlockReceiptsPrefix, hash.Bytes()...)) + if len(data) == 0 { + return nil + } } storageReceipts := []*types.ReceiptForStorage{} if err := rlp.DecodeBytes(data, &storageReceipts); err != nil { @@ -235,10 +285,9 @@ func GetReceipt(db ethdb.Database, txHash common.Hash) *types.Receipt { // WriteCanonicalHash stores the canonical hash for the given block number. func WriteCanonicalHash(db ethdb.Database, hash common.Hash, number uint64) error { - key := append(blockNumPrefix, big.NewInt(int64(number)).Bytes()...) + key := append(append(headerPrefix, encodeBlockNumber(number)...), numSuffix...) if err := db.Put(key, hash.Bytes()); err != nil { glog.Fatalf("failed to store number to hash mapping into database: %v", err) - return err } return nil } @@ -247,7 +296,6 @@ func WriteCanonicalHash(db ethdb.Database, hash common.Hash, number uint64) erro func WriteHeadHeaderHash(db ethdb.Database, hash common.Hash) error { if err := db.Put(headHeaderKey, hash.Bytes()); err != nil { glog.Fatalf("failed to store last header's hash into database: %v", err) - return err } return nil } @@ -256,7 +304,6 @@ func WriteHeadHeaderHash(db ethdb.Database, hash common.Hash) error { func WriteHeadBlockHash(db ethdb.Database, hash common.Hash) error { if err := db.Put(headBlockKey, hash.Bytes()); err != nil { glog.Fatalf("failed to store last block's hash into database: %v", err) - return err } return nil } @@ -265,7 +312,6 @@ func WriteHeadBlockHash(db ethdb.Database, hash common.Hash) error { func WriteHeadFastBlockHash(db ethdb.Database, hash common.Hash) error { if err := db.Put(headFastKey, hash.Bytes()); err != nil { glog.Fatalf("failed to store last fast block's hash into database: %v", err) - return err } return nil } @@ -276,40 +322,44 @@ func WriteHeader(db ethdb.Database, header *types.Header) error { if err != nil { return err } - key := append(append(blockPrefix, header.Hash().Bytes()...), headerSuffix...) + hash := header.Hash().Bytes() + num := header.Number.Uint64() + encNum := encodeBlockNumber(num) + key := append(blockHashPrefix, hash...) + if err := db.Put(key, encNum); err != nil { + glog.Fatalf("failed to store hash to number mapping into database: %v", err) + } + key = append(append(headerPrefix, encNum...), hash...) if err := db.Put(key, data); err != nil { glog.Fatalf("failed to store header into database: %v", err) - return err } - glog.V(logger.Debug).Infof("stored header #%v [%x…]", header.Number, header.Hash().Bytes()[:4]) + glog.V(logger.Debug).Infof("stored header #%v [%x…]", header.Number, hash[:4]) return nil } // WriteBody serializes the body of a block into the database. -func WriteBody(db ethdb.Database, hash common.Hash, body *types.Body) error { +func WriteBody(db ethdb.Database, hash common.Hash, number uint64, body *types.Body) error { data, err := rlp.EncodeToBytes(body) if err != nil { return err } - key := append(append(blockPrefix, hash.Bytes()...), bodySuffix...) + key := append(append(bodyPrefix, encodeBlockNumber(number)...), hash.Bytes()...) if err := db.Put(key, data); err != nil { glog.Fatalf("failed to store block body into database: %v", err) - return err } glog.V(logger.Debug).Infof("stored block body [%x…]", hash.Bytes()[:4]) return nil } // WriteTd serializes the total difficulty of a block into the database. -func WriteTd(db ethdb.Database, hash common.Hash, td *big.Int) error { +func WriteTd(db ethdb.Database, hash common.Hash, number uint64, td *big.Int) error { data, err := rlp.EncodeToBytes(td) if err != nil { return err } - key := append(append(blockPrefix, hash.Bytes()...), tdSuffix...) + key := append(append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...), tdSuffix...) if err := db.Put(key, data); err != nil { glog.Fatalf("failed to store block total difficulty into database: %v", err) - return err } glog.V(logger.Debug).Infof("stored block total difficulty [%x…]: %v", hash.Bytes()[:4], td) return nil @@ -318,7 +368,7 @@ func WriteTd(db ethdb.Database, hash common.Hash, td *big.Int) error { // WriteBlock serializes a block into the database, header and body separately. func WriteBlock(db ethdb.Database, block *types.Block) error { // Store the body first to retain database consistency - if err := WriteBody(db, block.Hash(), block.Body()); err != nil { + if err := WriteBody(db, block.Hash(), block.NumberU64(), block.Body()); err != nil { return err } // Store the header too, signaling full block ownership @@ -331,7 +381,7 @@ func WriteBlock(db ethdb.Database, block *types.Block) error { // WriteBlockReceipts stores all the transaction receipts belonging to a block // as a single receipt slice. This is used during chain reorganisations for // rescheduling dropped transactions. -func WriteBlockReceipts(db ethdb.Database, hash common.Hash, receipts types.Receipts) error { +func WriteBlockReceipts(db ethdb.Database, hash common.Hash, number uint64, receipts types.Receipts) error { // Convert the receipts into their storage form and serialize them storageReceipts := make([]*types.ReceiptForStorage, len(receipts)) for i, receipt := range receipts { @@ -342,9 +392,9 @@ func WriteBlockReceipts(db ethdb.Database, hash common.Hash, receipts types.Rece return err } // Store the flattened receipt slice - if err := db.Put(append(blockReceiptsPrefix, hash.Bytes()...), bytes); err != nil { + key := append(append(blockReceiptsPrefix, encodeBlockNumber(number)...), hash.Bytes()...) + if err := db.Put(key, bytes); err != nil { glog.Fatalf("failed to store block receipts into database: %v", err) - return err } glog.V(logger.Debug).Infof("stored block receipts [%x…]", hash.Bytes()[:4]) return nil @@ -388,7 +438,6 @@ func WriteTransactions(db ethdb.Database, block *types.Block) error { // Write the scheduled data into the database if err := batch.Write(); err != nil { glog.Fatalf("failed to store transactions into database: %v", err) - return err } return nil } @@ -411,42 +460,42 @@ func WriteReceipts(db ethdb.Database, receipts types.Receipts) error { // Write the scheduled data into the database if err := batch.Write(); err != nil { glog.Fatalf("failed to store receipts into database: %v", err) - return err } return nil } // DeleteCanonicalHash removes the number to hash canonical mapping. func DeleteCanonicalHash(db ethdb.Database, number uint64) { - db.Delete(append(blockNumPrefix, big.NewInt(int64(number)).Bytes()...)) + db.Delete(append(append(headerPrefix, encodeBlockNumber(number)...), numSuffix...)) } // DeleteHeader removes all block header data associated with a hash. -func DeleteHeader(db ethdb.Database, hash common.Hash) { - db.Delete(append(append(blockPrefix, hash.Bytes()...), headerSuffix...)) +func DeleteHeader(db ethdb.Database, hash common.Hash, number uint64) { + db.Delete(append(blockHashPrefix, hash.Bytes()...)) + db.Delete(append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...)) } // DeleteBody removes all block body data associated with a hash. -func DeleteBody(db ethdb.Database, hash common.Hash) { - db.Delete(append(append(blockPrefix, hash.Bytes()...), bodySuffix...)) +func DeleteBody(db ethdb.Database, hash common.Hash, number uint64) { + db.Delete(append(append(bodyPrefix, encodeBlockNumber(number)...), hash.Bytes()...)) } // DeleteTd removes all block total difficulty data associated with a hash. -func DeleteTd(db ethdb.Database, hash common.Hash) { - db.Delete(append(append(blockPrefix, hash.Bytes()...), tdSuffix...)) +func DeleteTd(db ethdb.Database, hash common.Hash, number uint64) { + db.Delete(append(append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...), tdSuffix...)) } // DeleteBlock removes all block data associated with a hash. -func DeleteBlock(db ethdb.Database, hash common.Hash) { - DeleteBlockReceipts(db, hash) - DeleteHeader(db, hash) - DeleteBody(db, hash) - DeleteTd(db, hash) +func DeleteBlock(db ethdb.Database, hash common.Hash, number uint64) { + DeleteBlockReceipts(db, hash, number) + DeleteHeader(db, hash, number) + DeleteBody(db, hash, number) + DeleteTd(db, hash, number) } // DeleteBlockReceipts removes all receipt data associated with a block hash. -func DeleteBlockReceipts(db ethdb.Database, hash common.Hash) { - db.Delete(append(blockReceiptsPrefix, hash.Bytes()...)) +func DeleteBlockReceipts(db ethdb.Database, hash common.Hash, number uint64) { + db.Delete(append(append(blockReceiptsPrefix, encodeBlockNumber(number)...), hash.Bytes()...)) } // DeleteTransaction removes all transaction data associated with a hash. @@ -466,7 +515,7 @@ func DeleteReceipt(db ethdb.Database, hash common.Hash) { // access the old combined block representation. It will be dropped after the // network transitions to eth/63. func GetBlockByHashOld(db ethdb.Database, hash common.Hash) *types.Block { - data, _ := db.Get(append(blockHashPrefix, hash[:]...)) + data, _ := db.Get(append(oldBlockHashPrefix, hash[:]...)) if len(data) == 0 { return nil } diff --git a/core/database_util_test.go b/core/database_util_test.go index 9ef787624..6c19f78c8 100644 --- a/core/database_util_test.go +++ b/core/database_util_test.go @@ -89,20 +89,20 @@ func TestHeaderStorage(t *testing.T) { db, _ := ethdb.NewMemDatabase() // Create a test header to move around the database and make sure it's really new - header := &types.Header{Extra: []byte("test header")} - if entry := GetHeader(db, header.Hash()); entry != nil { + header := &types.Header{Number: big.NewInt(42), Extra: []byte("test header")} + if entry := GetHeader(db, header.Hash(), header.Number.Uint64()); entry != nil { t.Fatalf("Non existent header returned: %v", entry) } // Write and verify the header in the database if err := WriteHeader(db, header); err != nil { t.Fatalf("Failed to write header into database: %v", err) } - if entry := GetHeader(db, header.Hash()); entry == nil { + if entry := GetHeader(db, header.Hash(), header.Number.Uint64()); entry == nil { t.Fatalf("Stored header not found") } else if entry.Hash() != header.Hash() { t.Fatalf("Retrieved header mismatch: have %v, want %v", entry, header) } - if entry := GetHeaderRLP(db, header.Hash()); entry == nil { + if entry := GetHeaderRLP(db, header.Hash(), header.Number.Uint64()); entry == nil { t.Fatalf("Stored header RLP not found") } else { hasher := sha3.NewKeccak256() @@ -113,8 +113,8 @@ func TestHeaderStorage(t *testing.T) { } } // Delete the header and verify the execution - DeleteHeader(db, header.Hash()) - if entry := GetHeader(db, header.Hash()); entry != nil { + DeleteHeader(db, header.Hash(), header.Number.Uint64()) + if entry := GetHeader(db, header.Hash(), header.Number.Uint64()); entry != nil { t.Fatalf("Deleted header returned: %v", entry) } } @@ -130,19 +130,19 @@ func TestBodyStorage(t *testing.T) { rlp.Encode(hasher, body) hash := common.BytesToHash(hasher.Sum(nil)) - if entry := GetBody(db, hash); entry != nil { + if entry := GetBody(db, hash, 0); entry != nil { t.Fatalf("Non existent body returned: %v", entry) } // Write and verify the body in the database - if err := WriteBody(db, hash, body); err != nil { + if err := WriteBody(db, hash, 0, body); err != nil { t.Fatalf("Failed to write body into database: %v", err) } - if entry := GetBody(db, hash); entry == nil { + if entry := GetBody(db, hash, 0); entry == nil { t.Fatalf("Stored body not found") } else if types.DeriveSha(types.Transactions(entry.Transactions)) != types.DeriveSha(types.Transactions(body.Transactions)) || types.CalcUncleHash(entry.Uncles) != types.CalcUncleHash(body.Uncles) { t.Fatalf("Retrieved body mismatch: have %v, want %v", entry, body) } - if entry := GetBodyRLP(db, hash); entry == nil { + if entry := GetBodyRLP(db, hash, 0); entry == nil { t.Fatalf("Stored body RLP not found") } else { hasher := sha3.NewKeccak256() @@ -153,8 +153,8 @@ func TestBodyStorage(t *testing.T) { } } // Delete the body and verify the execution - DeleteBody(db, hash) - if entry := GetBody(db, hash); entry != nil { + DeleteBody(db, hash, 0) + if entry := GetBody(db, hash, 0); entry != nil { t.Fatalf("Deleted body returned: %v", entry) } } @@ -170,43 +170,43 @@ func TestBlockStorage(t *testing.T) { TxHash: types.EmptyRootHash, ReceiptHash: types.EmptyRootHash, }) - if entry := GetBlock(db, block.Hash()); entry != nil { + if entry := GetBlock(db, block.Hash(), block.NumberU64()); entry != nil { t.Fatalf("Non existent block returned: %v", entry) } - if entry := GetHeader(db, block.Hash()); entry != nil { + if entry := GetHeader(db, block.Hash(), block.NumberU64()); entry != nil { t.Fatalf("Non existent header returned: %v", entry) } - if entry := GetBody(db, block.Hash()); entry != nil { + if entry := GetBody(db, block.Hash(), block.NumberU64()); entry != nil { t.Fatalf("Non existent body returned: %v", entry) } // Write and verify the block in the database if err := WriteBlock(db, block); err != nil { t.Fatalf("Failed to write block into database: %v", err) } - if entry := GetBlock(db, block.Hash()); entry == nil { + if entry := GetBlock(db, block.Hash(), block.NumberU64()); entry == nil { t.Fatalf("Stored block not found") } else if entry.Hash() != block.Hash() { t.Fatalf("Retrieved block mismatch: have %v, want %v", entry, block) } - if entry := GetHeader(db, block.Hash()); entry == nil { + if entry := GetHeader(db, block.Hash(), block.NumberU64()); entry == nil { t.Fatalf("Stored header not found") } else if entry.Hash() != block.Header().Hash() { t.Fatalf("Retrieved header mismatch: have %v, want %v", entry, block.Header()) } - if entry := GetBody(db, block.Hash()); entry == nil { + if entry := GetBody(db, block.Hash(), block.NumberU64()); entry == nil { t.Fatalf("Stored body not found") } else if types.DeriveSha(types.Transactions(entry.Transactions)) != types.DeriveSha(block.Transactions()) || types.CalcUncleHash(entry.Uncles) != types.CalcUncleHash(block.Uncles()) { t.Fatalf("Retrieved body mismatch: have %v, want %v", entry, block.Body()) } // Delete the block and verify the execution - DeleteBlock(db, block.Hash()) - if entry := GetBlock(db, block.Hash()); entry != nil { + DeleteBlock(db, block.Hash(), block.NumberU64()) + if entry := GetBlock(db, block.Hash(), block.NumberU64()); entry != nil { t.Fatalf("Deleted block returned: %v", entry) } - if entry := GetHeader(db, block.Hash()); entry != nil { + if entry := GetHeader(db, block.Hash(), block.NumberU64()); entry != nil { t.Fatalf("Deleted header returned: %v", entry) } - if entry := GetBody(db, block.Hash()); entry != nil { + if entry := GetBody(db, block.Hash(), block.NumberU64()); entry != nil { t.Fatalf("Deleted body returned: %v", entry) } } @@ -224,28 +224,28 @@ func TestPartialBlockStorage(t *testing.T) { if err := WriteHeader(db, block.Header()); err != nil { t.Fatalf("Failed to write header into database: %v", err) } - if entry := GetBlock(db, block.Hash()); entry != nil { + if entry := GetBlock(db, block.Hash(), block.NumberU64()); entry != nil { t.Fatalf("Non existent block returned: %v", entry) } - DeleteHeader(db, block.Hash()) + DeleteHeader(db, block.Hash(), block.NumberU64()) // Store a body and check that it's not recognized as a block - if err := WriteBody(db, block.Hash(), block.Body()); err != nil { + if err := WriteBody(db, block.Hash(), block.NumberU64(), block.Body()); err != nil { t.Fatalf("Failed to write body into database: %v", err) } - if entry := GetBlock(db, block.Hash()); entry != nil { + if entry := GetBlock(db, block.Hash(), block.NumberU64()); entry != nil { t.Fatalf("Non existent block returned: %v", entry) } - DeleteBody(db, block.Hash()) + DeleteBody(db, block.Hash(), block.NumberU64()) // Store a header and a body separately and check reassembly if err := WriteHeader(db, block.Header()); err != nil { t.Fatalf("Failed to write header into database: %v", err) } - if err := WriteBody(db, block.Hash(), block.Body()); err != nil { + if err := WriteBody(db, block.Hash(), block.NumberU64(), block.Body()); err != nil { t.Fatalf("Failed to write body into database: %v", err) } - if entry := GetBlock(db, block.Hash()); entry == nil { + if entry := GetBlock(db, block.Hash(), block.NumberU64()); entry == nil { t.Fatalf("Stored block not found") } else if entry.Hash() != block.Hash() { t.Fatalf("Retrieved block mismatch: have %v, want %v", entry, block) @@ -258,21 +258,21 @@ func TestTdStorage(t *testing.T) { // Create a test TD to move around the database and make sure it's really new hash, td := common.Hash{}, big.NewInt(314) - if entry := GetTd(db, hash); entry != nil { + if entry := GetTd(db, hash, 0); entry != nil { t.Fatalf("Non existent TD returned: %v", entry) } // Write and verify the TD in the database - if err := WriteTd(db, hash, td); err != nil { + if err := WriteTd(db, hash, 0, td); err != nil { t.Fatalf("Failed to write TD into database: %v", err) } - if entry := GetTd(db, hash); entry == nil { + if entry := GetTd(db, hash, 0); entry == nil { t.Fatalf("Stored TD not found") } else if entry.Cmp(td) != 0 { t.Fatalf("Retrieved TD mismatch: have %v, want %v", entry, td) } // Delete the TD and verify the execution - DeleteTd(db, hash) - if entry := GetTd(db, hash); entry != nil { + DeleteTd(db, hash, 0) + if entry := GetTd(db, hash, 0); entry != nil { t.Fatalf("Deleted TD returned: %v", entry) } } @@ -473,14 +473,14 @@ func TestBlockReceiptStorage(t *testing.T) { // Check that no receipt entries are in a pristine database hash := common.BytesToHash([]byte{0x03, 0x14}) - if rs := GetBlockReceipts(db, hash); len(rs) != 0 { + if rs := GetBlockReceipts(db, hash, 0); len(rs) != 0 { t.Fatalf("non existent receipts returned: %v", rs) } // Insert the receipt slice into the database and check presence - if err := WriteBlockReceipts(db, hash, receipts); err != nil { + if err := WriteBlockReceipts(db, hash, 0, receipts); err != nil { t.Fatalf("failed to write block receipts: %v", err) } - if rs := GetBlockReceipts(db, hash); len(rs) == 0 { + if rs := GetBlockReceipts(db, hash, 0); len(rs) == 0 { t.Fatalf("no receipts returned") } else { for i := 0; i < len(receipts); i++ { @@ -493,8 +493,8 @@ func TestBlockReceiptStorage(t *testing.T) { } } // Delete the receipt slice and check purge - DeleteBlockReceipts(db, hash) - if rs := GetBlockReceipts(db, hash); len(rs) != 0 { + DeleteBlockReceipts(db, hash, 0) + if rs := GetBlockReceipts(db, hash, 0); len(rs) != 0 { t.Fatalf("deleted receipts returned: %v", rs) } } @@ -597,7 +597,7 @@ func TestMipmapChain(t *testing.T) { if err := WriteHeadBlockHash(db, block.Hash()); err != nil { t.Fatalf("failed to insert block number: %v", err) } - if err := WriteBlockReceipts(db, block.Hash(), receipts[i]); err != nil { + if err := WriteBlockReceipts(db, block.Hash(), block.NumberU64(), receipts[i]); err != nil { t.Fatal("error writing block receipts:", err) } } diff --git a/core/genesis.go b/core/genesis.go index 40d799621..637b63320 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -88,7 +88,7 @@ func WriteGenesisBlock(chainDb ethdb.Database, reader io.Reader) (*types.Block, Root: root, }, nil, nil, nil) - if block := GetBlock(chainDb, block.Hash()); block != nil { + if block := GetBlock(chainDb, block.Hash(), block.NumberU64()); block != nil { glog.V(logger.Info).Infoln("Genesis block already in chain. Writing canonical number") err := WriteCanonicalHash(chainDb, block.Hash(), block.NumberU64()) if err != nil { @@ -100,13 +100,13 @@ func WriteGenesisBlock(chainDb ethdb.Database, reader io.Reader) (*types.Block, if err := stateBatch.Write(); err != nil { return nil, fmt.Errorf("cannot write state: %v", err) } - if err := WriteTd(chainDb, block.Hash(), difficulty); err != nil { + if err := WriteTd(chainDb, block.Hash(), block.NumberU64(), difficulty); err != nil { return nil, err } if err := WriteBlock(chainDb, block); err != nil { return nil, err } - if err := WriteBlockReceipts(chainDb, block.Hash(), nil); err != nil { + if err := WriteBlockReceipts(chainDb, block.Hash(), block.NumberU64(), nil); err != nil { return nil, err } if err := WriteCanonicalHash(chainDb, block.Hash(), block.NumberU64()); err != nil { diff --git a/core/headerchain.go b/core/headerchain.go index 5e0fbfb08..f856333a0 100644 --- a/core/headerchain.go +++ b/core/headerchain.go @@ -35,6 +35,12 @@ import ( "github.com/hashicorp/golang-lru" ) +const ( + headerCacheLimit = 512 + tdCacheLimit = 1024 + numberCacheLimit = 2048 +) + // HeaderChain implements the basic block header chain logic that is shared by // core.BlockChain and light.LightChain. It is not usable in itself, only as // a part of either structure. @@ -51,6 +57,7 @@ type HeaderChain struct { headerCache *lru.Cache // Cache for the most recent block headers tdCache *lru.Cache // Cache for the most recent block total difficulties + numberCache *lru.Cache // Cache for the most recent block numbers procInterrupt func() bool @@ -68,6 +75,7 @@ type getHeaderValidatorFn func() HeaderValidator func NewHeaderChain(chainDb ethdb.Database, config *ChainConfig, getValidator getHeaderValidatorFn, procInterrupt func() bool) (*HeaderChain, error) { headerCache, _ := lru.New(headerCacheLimit) tdCache, _ := lru.New(tdCacheLimit) + numberCache, _ := lru.New(numberCacheLimit) // Seed a fast but crypto originating random generator seed, err := crand.Int(crand.Reader, big.NewInt(math.MaxInt64)) @@ -80,6 +88,7 @@ func NewHeaderChain(chainDb ethdb.Database, config *ChainConfig, getValidator ge chainDb: chainDb, headerCache: headerCache, tdCache: tdCache, + numberCache: numberCache, procInterrupt: procInterrupt, rand: mrand.New(mrand.NewSource(seed.Int64())), getValidator: getValidator, @@ -97,7 +106,7 @@ func NewHeaderChain(chainDb ethdb.Database, config *ChainConfig, getValidator ge hc.currentHeader = hc.genesisHeader if head := GetHeadBlockHash(chainDb); head != (common.Hash{}) { - if chead := hc.GetHeader(head); chead != nil { + if chead := hc.GetHeaderByHash(head); chead != nil { hc.currentHeader = chead } } @@ -106,6 +115,19 @@ func NewHeaderChain(chainDb ethdb.Database, config *ChainConfig, getValidator ge return hc, nil } +// GetBlockNumber retrieves the block number belonging to the given hash +// from the cache or database +func (hc *HeaderChain) GetBlockNumber(hash common.Hash) uint64 { + if cached, ok := hc.numberCache.Get(hash); ok { + return cached.(uint64) + } + number := GetBlockNumber(hc.chainDb, hash) + if number != missingNumber { + hc.numberCache.Add(hash, number) + } + return number +} + // WriteHeader writes a header into the local chain, given that its parent is // already known. If the total difficulty of the newly inserted header becomes // greater than the current known TD, the canonical chain is re-routed. @@ -122,11 +144,11 @@ func (hc *HeaderChain) WriteHeader(header *types.Header) (status WriteStatus, er number = header.Number.Uint64() ) // Calculate the total difficulty of the header - ptd := hc.GetTd(header.ParentHash) + ptd := hc.GetTd(header.ParentHash, number-1) if ptd == nil { return NonStatTy, ParentError(header.ParentHash) } - localTd := hc.GetTd(hc.currentHeaderHash) + localTd := hc.GetTd(hc.currentHeaderHash, hc.currentHeader.Number.Uint64()) externTd := new(big.Int).Add(header.Difficulty, ptd) // If the total difficulty is higher than our known, add it to the canonical chain @@ -134,21 +156,25 @@ func (hc *HeaderChain) WriteHeader(header *types.Header) (status WriteStatus, er // Please refer to http://www.cs.cornell.edu/~ie53/publications/btcProcFC.pdf if externTd.Cmp(localTd) > 0 || (externTd.Cmp(localTd) == 0 && mrand.Float64() < 0.5) { // Delete any canonical number assignments above the new head - for i := number + 1; GetCanonicalHash(hc.chainDb, i) != (common.Hash{}); i++ { + for i := number + 1; ; i++ { + hash := GetCanonicalHash(hc.chainDb, i) + if hash == (common.Hash{}) { + break + } DeleteCanonicalHash(hc.chainDb, i) } // Overwrite any stale canonical number assignments var ( headHash = header.ParentHash - headHeader = hc.GetHeader(headHash) - headNumber = headHeader.Number.Uint64() + headNumber = header.Number.Uint64() - 1 + headHeader = hc.GetHeader(headHash, headNumber) ) for GetCanonicalHash(hc.chainDb, headNumber) != headHash { WriteCanonicalHash(hc.chainDb, headHash, headNumber) headHash = headHeader.ParentHash - headHeader = hc.GetHeader(headHash) - headNumber = headHeader.Number.Uint64() + headNumber = headHeader.Number.Uint64() - 1 + headHeader = hc.GetHeader(headHash, headNumber) } // Extend the canonical chain with the new header if err := WriteCanonicalHash(hc.chainDb, hash, number); err != nil { @@ -164,13 +190,14 @@ func (hc *HeaderChain) WriteHeader(header *types.Header) (status WriteStatus, er status = SideStatTy } // Irrelevant of the canonical status, write the header itself to the database - if err := hc.WriteTd(hash, externTd); err != nil { + if err := hc.WriteTd(hash, number, externTd); err != nil { glog.Fatalf("failed to write header total difficulty: %v", err) } if err := WriteHeader(hc.chainDb, header); err != nil { glog.Fatalf("failed to write header contents: %v", err) } hc.headerCache.Add(hash, header) + hc.numberCache.Add(hash, number) return } @@ -239,7 +266,7 @@ func (hc *HeaderChain) InsertHeaderChain(chain []*types.Header, checkFreq int, w var err error if index == 0 { - err = hc.getValidator().ValidateHeader(header, hc.GetHeader(header.ParentHash), checkPow) + err = hc.getValidator().ValidateHeader(header, hc.GetHeader(header.ParentHash, header.Number.Uint64()-1), checkPow) } else { err = hc.getValidator().ValidateHeader(header, chain[index-1], checkPow) } @@ -300,7 +327,7 @@ func (hc *HeaderChain) InsertHeaderChain(chain []*types.Header, checkFreq int, w // hash, fetching towards the genesis block. func (hc *HeaderChain) GetBlockHashesFromHash(hash common.Hash, max uint64) []common.Hash { // Get the origin header from which to fetch - header := hc.GetHeader(hash) + header := hc.GetHeaderByHash(hash) if header == nil { return nil } @@ -308,7 +335,7 @@ func (hc *HeaderChain) GetBlockHashesFromHash(hash common.Hash, max uint64) []co chain := make([]common.Hash, 0, max) for i := uint64(0); i < max; i++ { next := header.ParentHash - if header = hc.GetHeader(next); header == nil { + if header = hc.GetHeader(next, header.Number.Uint64()-1); header == nil { break } chain = append(chain, next) @@ -320,13 +347,13 @@ func (hc *HeaderChain) GetBlockHashesFromHash(hash common.Hash, max uint64) []co } // GetTd retrieves a block's total difficulty in the canonical chain from the -// database by hash, caching it if found. -func (hc *HeaderChain) GetTd(hash common.Hash) *big.Int { +// database by hash and number, caching it if found. +func (hc *HeaderChain) GetTd(hash common.Hash, number uint64) *big.Int { // Short circuit if the td's already in the cache, retrieve otherwise if cached, ok := hc.tdCache.Get(hash); ok { return cached.(*big.Int) } - td := GetTd(hc.chainDb, hash) + td := GetTd(hc.chainDb, hash, number) if td == nil { return nil } @@ -335,24 +362,30 @@ func (hc *HeaderChain) GetTd(hash common.Hash) *big.Int { return td } +// GetTdByHash retrieves a block's total difficulty in the canonical chain from the +// database by hash, caching it if found. +func (hc *HeaderChain) GetTdByHash(hash common.Hash) *big.Int { + return hc.GetTd(hash, hc.GetBlockNumber(hash)) +} + // WriteTd stores a block's total difficulty into the database, also caching it // along the way. -func (hc *HeaderChain) WriteTd(hash common.Hash, td *big.Int) error { - if err := WriteTd(hc.chainDb, hash, td); err != nil { +func (hc *HeaderChain) WriteTd(hash common.Hash, number uint64, td *big.Int) error { + if err := WriteTd(hc.chainDb, hash, number, td); err != nil { return err } hc.tdCache.Add(hash, new(big.Int).Set(td)) return nil } -// GetHeader retrieves a block header from the database by hash, caching it if -// found. -func (hc *HeaderChain) GetHeader(hash common.Hash) *types.Header { +// GetHeader retrieves a block header from the database by hash and number, +// caching it if found. +func (hc *HeaderChain) GetHeader(hash common.Hash, number uint64) *types.Header { // Short circuit if the header's already in the cache, retrieve otherwise if header, ok := hc.headerCache.Get(hash); ok { return header.(*types.Header) } - header := GetHeader(hc.chainDb, hash) + header := GetHeader(hc.chainDb, hash, number) if header == nil { return nil } @@ -361,10 +394,16 @@ func (hc *HeaderChain) GetHeader(hash common.Hash) *types.Header { return header } +// GetHeaderByHash retrieves a block header from the database by hash, caching it if +// found. +func (hc *HeaderChain) GetHeaderByHash(hash common.Hash) *types.Header { + return hc.GetHeader(hash, hc.GetBlockNumber(hash)) +} + // HasHeader checks if a block header is present in the database or not, caching // it if present. func (hc *HeaderChain) HasHeader(hash common.Hash) bool { - return hc.GetHeader(hash) != nil + return hc.GetHeaderByHash(hash) != nil } // GetHeaderByNumber retrieves a block header from the database by number, @@ -374,7 +413,7 @@ func (hc *HeaderChain) GetHeaderByNumber(number uint64) *types.Header { if hash == (common.Hash{}) { return nil } - return hc.GetHeader(hash) + return hc.GetHeader(hash, number) } // CurrentHeader retrieves the current head header of the canonical chain. The @@ -394,7 +433,7 @@ func (hc *HeaderChain) SetCurrentHeader(head *types.Header) { // DeleteCallback is a callback function that is called by SetHead before // each header is deleted. -type DeleteCallback func(common.Hash) +type DeleteCallback func(common.Hash, uint64) // SetHead rewinds the local chain to a new head. Everything above the new head // will be deleted and the new one set. @@ -406,12 +445,13 @@ func (hc *HeaderChain) SetHead(head uint64, delFn DeleteCallback) { for hc.currentHeader != nil && hc.currentHeader.Number.Uint64() > head { hash := hc.currentHeader.Hash() + num := hc.currentHeader.Number.Uint64() if delFn != nil { - delFn(hash) + delFn(hash, num) } - DeleteHeader(hc.chainDb, hash) - DeleteTd(hc.chainDb, hash) - hc.currentHeader = hc.GetHeader(hc.currentHeader.ParentHash) + DeleteHeader(hc.chainDb, hash, num) + DeleteTd(hc.chainDb, hash, num) + hc.currentHeader = hc.GetHeader(hc.currentHeader.ParentHash, hc.currentHeader.Number.Uint64()-1) } // Roll back the canonical chain numbering for i := height; i > head; i-- { @@ -420,6 +460,7 @@ func (hc *HeaderChain) SetHead(head uint64, delFn DeleteCallback) { // Clear out any stale content from the caches hc.headerCache.Purge() hc.tdCache.Purge() + hc.numberCache.Purge() if hc.currentHeader == nil { hc.currentHeader = hc.genesisHeader diff --git a/core/vm_env.go b/core/vm_env.go index f50140c68..599672382 100644 --- a/core/vm_env.go +++ b/core/vm_env.go @@ -30,7 +30,7 @@ import ( // to query for information. func GetHashFn(ref common.Hash, chain *BlockChain) func(n uint64) common.Hash { return func(n uint64) common.Hash { - for block := chain.GetBlock(ref); block != nil; block = chain.GetBlock(block.ParentHash()) { + for block := chain.GetBlockByHash(ref); block != nil; block = chain.GetBlock(block.ParentHash(), block.NumberU64()-1) { if block.NumberU64() == n { return block.Hash() } |