all: nuke total difficulty

cmd/devp2p/internal/ethtest/chain.go

@@ -143,11 +143,7 @@ func (c *Chain) ForkID() forkid.ID {
 // TD calculates the total difficulty of the chain at the
 // chain head.
 func (c *Chain) TD() *big.Int {
-	sum := new(big.Int)
-	for _, block := range c.blocks[:c.Len()] {
-		sum.Add(sum, block.Difficulty())
-	}
-	return sum
+	return new(big.Int)
 }
 
 // GetBlock returns the block at the specified number.

cmd/devp2p/internal/ethtest/suite.go

@@ -18,7 +18,6 @@ package ethtest
 
 import (
 	"crypto/rand"
-	"math/big"
 	"reflect"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -74,7 +73,6 @@ func (s *Suite) EthTests() []utesting.Test {
 		{Name: "GetBlockBodies", Fn: s.TestGetBlockBodies},
 		// // malicious handshakes + status
 		{Name: "MaliciousHandshake", Fn: s.TestMaliciousHandshake},
-		{Name: "MaliciousStatus", Fn: s.TestMaliciousStatus},
 		// test transactions
 		{Name: "LargeTxRequest", Fn: s.TestLargeTxRequest, Slow: true},
 		{Name: "Transaction", Fn: s.TestTransaction},
@@ -453,42 +451,6 @@ func (s *Suite) TestMaliciousHandshake(t *utesting.T) {
 	}
 }
 
-func (s *Suite) TestMaliciousStatus(t *utesting.T) {
-	t.Log(`This test sends a malicious eth Status message to the node and expects a disconnect.`)
-
-	conn, err := s.dial()
-	if err != nil {
-		t.Fatalf("dial failed: %v", err)
-	}
-	defer conn.Close()
-	if err := conn.handshake(); err != nil {
-		t.Fatalf("handshake failed: %v", err)
-	}
-	// Create status with large total difficulty.
-	status := &eth.StatusPacket{
-		ProtocolVersion: uint32(conn.negotiatedProtoVersion),
-		NetworkID:       s.chain.config.ChainID.Uint64(),
-		TD:              new(big.Int).SetBytes(randBuf(2048)),
-		Head:            s.chain.Head().Hash(),
-		Genesis:         s.chain.GetBlock(0).Hash(),
-		ForkID:          s.chain.ForkID(),
-	}
-	if err := conn.statusExchange(s.chain, status); err != nil {
-		t.Fatalf("status exchange failed: %v", err)
-	}
-	// Wait for disconnect.
-	code, _, err := conn.Read()
-	if err != nil {
-		t.Fatalf("error reading from connection: %v", err)
-	}
-	switch code {
-	case discMsg:
-		break
-	default:
-		t.Fatalf("expected disconnect, got: %d", code)
-	}
-}
-
 func (s *Suite) TestTransaction(t *utesting.T) {
 	t.Log(`This test sends a valid transaction to the node and checks if the
 transaction gets propagated.`)

cmd/utils/cmd.go

@@ -25,6 +25,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"math/big"
 	"os"
 	"os/signal"
 	"path/filepath"
@@ -422,6 +423,10 @@ func ExportHistory(bc *core.BlockChain, dir string, first, last, step uint64) er
 		buf       = bytes.NewBuffer(nil)
 		checksums []string
 	)
+	td := new(big.Int)
+	for i := uint64(0); i < first; i++ {
+		td.Add(td, bc.GetHeaderByNumber(i).Difficulty)
+	}
 	for i := first; i <= last; i += step {
 		err := func() error {
 			filename := filepath.Join(dir, era.Filename(network, int(i/step), common.Hash{}))
@@ -444,10 +449,7 @@ func ExportHistory(bc *core.BlockChain, dir string, first, last, step uint64) er
 				if receipts == nil {
 					return fmt.Errorf("export failed on #%d: receipts not found", n)
 				}
-				td := bc.GetTd(block.Hash(), block.NumberU64())
-				if td == nil {
-					return fmt.Errorf("export failed on #%d: total difficulty not found", n)
-				}
+				td.Add(td, block.Difficulty())
 				if err := w.Add(block, receipts, td); err != nil {
 					return err
 				}

consensus/beacon/consensus.go

@@ -61,7 +61,8 @@ var (
 // is only used for necessary consensus checks. The legacy consensus engine can be any
 // engine implements the consensus interface (except the beacon itself).
 type Beacon struct {
-	ethone consensus.Engine // Original consensus engine used in eth1, e.g. ethash or clique
+	ethone   consensus.Engine // Original consensus engine used in eth1, e.g. ethash or clique
+	ttdblock *uint64          // Merge block-number for testchain generation without TTDs
 }
 
 // New creates a consensus engine with the given embedded eth1 engine.
@@ -72,6 +73,18 @@ func New(ethone consensus.Engine) *Beacon {
 	return &Beacon{ethone: ethone}
 }
 
+// TestingTTDBlock is a replacement mechanism for TTD-based pre-/post-merge
+// splitting. With chain history deletion, TD calculations become impossible.
+// This is fine for progressing the live chain, but to be able to generate test
+// chains, we do need a split point. This method supports setting an explicit
+// block number to use as the splitter *for testing*, instead of having to keep
+// the notion of TDs in the client just for testing.
+//
+// The block with supplied number is regarded as the last pre-merge block.
+func (beacon *Beacon) TestingTTDBlock(number uint64) {
+	beacon.ttdblock = &number
+}
+
 // Author implements consensus.Engine, returning the verified author of the block.
 func (beacon *Beacon) Author(header *types.Header) (common.Address, error) {
 	if !beacon.IsPoSHeader(header) {
@@ -83,78 +96,63 @@ func (beacon *Beacon) Author(header *types.Header) (common.Address, error) {
 // VerifyHeader checks whether a header conforms to the consensus rules of the
 // stock Ethereum consensus engine.
 func (beacon *Beacon) VerifyHeader(chain consensus.ChainHeaderReader, header *types.Header) error {
-	reached, err := IsTTDReached(chain, header.ParentHash, header.Number.Uint64()-1)
-	if err != nil {
-		return err
-	}
-	if !reached {
-		return beacon.ethone.VerifyHeader(chain, header)
-	}
-	// Short circuit if the parent is not known
+	// During the live merge transition, the consensus engine used the terminal
+	// total difficulty to detect when PoW (PoA) switched to PoS. Maintaining the
+	// total difficulty values however require applying all the blocks from the
+	// genesis to build up the TD. This stops being a possibility if the tail of
+	// the chain is pruned already during sync.
+	//
+	// One heuristic that can be used to distinguish pre-merge and post-merge
+	// blocks is whether their *difficulty* is >0 or ==0 respectively. This of
+	// course would mean that we cannot prove anymore for a past chain that it
+	// truly transitioned at the correct TTD, but if we consider that ancient
+	// point in time finalized a long time ago, there should be no attempt from
+	// the consensus client to rewrite very old history.
+	//
+	// One thing that's probably not needed but which we can add to make this
+	// verification even stricter is to enforce that the chain can switch from
+	// >0 to ==0 TD only once by forbidding an ==0 to be followed by a >0.
+
+	// Verify that we're not reverting to pre-merge from post-merge
 	parent := chain.GetHeader(header.ParentHash, header.Number.Uint64()-1)
 	if parent == nil {
 		return consensus.ErrUnknownAncestor
 	}
-	// Sanity checks passed, do a proper verification
-	return beacon.verifyHeader(chain, header, parent)
-}
-
-// errOut constructs an error channel with prefilled errors inside.
-func errOut(n int, err error) chan error {
-	errs := make(chan error, n)
-	for i := 0; i < n; i++ {
-		errs <- err
+	if parent.Difficulty.Sign() == 0 && header.Difficulty.Sign() > 0 {
+		return consensus.ErrInvalidTerminalBlock
+	}
+	// Check >0 TDs with pre-merge, --0 TDs with post-merge rules
+	if header.Difficulty.Sign() > 0 {
+		return beacon.ethone.VerifyHeader(chain, header)
 	}
-	return errs
+	return beacon.verifyHeader(chain, header, parent)
 }
 
 // splitHeaders splits the provided header batch into two parts according to
-// the configured ttd. It requires the parent of header batch along with its
-// td are stored correctly in chain. If ttd is not configured yet, all headers
-// will be treated legacy PoW headers.
+// the difficulty field.
+//
 // Note, this function will not verify the header validity but just split them.
-func (beacon *Beacon) splitHeaders(chain consensus.ChainHeaderReader, headers []*types.Header) ([]*types.Header, []*types.Header, error) {
-	// TTD is not defined yet, all headers should be in legacy format.
-	ttd := chain.Config().TerminalTotalDifficulty
-	ptd := chain.GetTd(headers[0].ParentHash, headers[0].Number.Uint64()-1)
-	if ptd == nil {
-		return nil, nil, consensus.ErrUnknownAncestor
-	}
-	// The entire header batch already crosses the transition.
-	if ptd.Cmp(ttd) >= 0 {
-		return nil, headers, nil
-	}
+func (beacon *Beacon) splitHeaders(headers []*types.Header) ([]*types.Header, []*types.Header) {
 	var (
 		preHeaders  = headers
 		postHeaders []*types.Header
-		td          = new(big.Int).Set(ptd)
-		tdPassed    bool
 	)
 	for i, header := range headers {
-		if tdPassed {
+		if header.Difficulty.Sign() == 0 {
 			preHeaders = headers[:i]
 			postHeaders = headers[i:]
 			break
 		}
-		td = td.Add(td, header.Difficulty)
-		if td.Cmp(ttd) >= 0 {
-			// This is the last PoW header, it still belongs to
-			// the preHeaders, so we cannot split+break yet.
-			tdPassed = true
-		}
 	}
-	return preHeaders, postHeaders, nil
+	return preHeaders, postHeaders
 }
 
 // VerifyHeaders is similar to VerifyHeader, but verifies a batch of headers
 // concurrently. The method returns a quit channel to abort the operations and
 // a results channel to retrieve the async verifications.
 // VerifyHeaders expect the headers to be ordered and continuous.
 func (beacon *Beacon) VerifyHeaders(chain consensus.ChainHeaderReader, headers []*types.Header) (chan<- struct{}, <-chan error) {
-	preHeaders, postHeaders, err := beacon.splitHeaders(chain, headers)
-	if err != nil {
-		return make(chan struct{}), errOut(len(headers), err)
-	}
+	preHeaders, postHeaders := beacon.splitHeaders(headers)
 	if len(postHeaders) == 0 {
 		return beacon.ethone.VerifyHeaders(chain, headers)
 	}
@@ -334,12 +332,15 @@ func (beacon *Beacon) verifyHeaders(chain consensus.ChainHeaderReader, headers [
 // Prepare implements consensus.Engine, initializing the difficulty field of a
 // header to conform to the beacon protocol. The changes are done inline.
 func (beacon *Beacon) Prepare(chain consensus.ChainHeaderReader, header *types.Header) error {
-	// Transition isn't triggered yet, use the legacy rules for preparation.
-	reached, err := IsTTDReached(chain, header.ParentHash, header.Number.Uint64()-1)
-	if err != nil {
-		return err
-	}
-	if !reached {
+	// The beacon engine requires access to total difficulties to be able to
+	// seal pre-merge and post-merge blocks. With the transition to removing
+	// old blocks, TDs become unaccessible, thus making TTD based pre-/post-
+	// merge decisions impossible.
+	//
+	// We do not need to seal non-merge blocks anymore live, but we do need
+	// to be able to generate test chains, thus we're reverting to a testing-
+	// settable field to direct that.
+	if beacon.ttdblock != nil && *beacon.ttdblock >= header.Number.Uint64() {
 		return beacon.ethone.Prepare(chain, header)
 	}
 	header.Difficulty = beaconDifficulty
@@ -449,8 +450,15 @@ func (beacon *Beacon) SealHash(header *types.Header) common.Hash {
 // the difficulty that a new block should have when created at time
 // given the parent block's time and difficulty.
 func (beacon *Beacon) CalcDifficulty(chain consensus.ChainHeaderReader, time uint64, parent *types.Header) *big.Int {
-	// Transition isn't triggered yet, use the legacy rules for calculation
-	if reached, _ := IsTTDReached(chain, parent.Hash(), parent.Number.Uint64()); !reached {
+	// The beacon engine requires access to total difficulties to be able to
+	// seal pre-merge and post-merge blocks. With the transition to removing
+	// old blocks, TDs become unaccessible, thus making TTD based pre-/post-
+	// merge decisions impossible.
+	//
+	// We do not need to seal non-merge blocks anymore live, but we do need
+	// to be able to generate test chains, thus we're reverting to a testing-
+	// settable field to direct that.
+	if beacon.ttdblock != nil && *beacon.ttdblock > parent.Number.Uint64() {
 		return beacon.ethone.CalcDifficulty(chain, time, parent)
 	}
 	return beaconDifficulty
@@ -491,14 +499,3 @@ func (beacon *Beacon) SetThreads(threads int) {
 		th.SetThreads(threads)
 	}
 }
-
-// IsTTDReached checks if the TotalTerminalDifficulty has been surpassed on the `parentHash` block.
-// It depends on the parentHash already being stored in the database.
-// If the parentHash is not stored in the database a UnknownAncestor error is returned.
-func IsTTDReached(chain consensus.ChainHeaderReader, parentHash common.Hash, parentNumber uint64) (bool, error) {
-	td := chain.GetTd(parentHash, parentNumber)
-	if td == nil {
-		return false, consensus.ErrUnknownAncestor
-	}
-	return td.Cmp(chain.Config().TerminalTotalDifficulty) >= 0, nil
-}

consensus/consensus.go

@@ -45,9 +45,6 @@ type ChainHeaderReader interface {
 
 	// GetHeaderByHash retrieves a block header from the database by its hash.
 	GetHeaderByHash(hash common.Hash) *types.Header
-
-	// GetTd retrieves the total difficulty from the database by hash and number.
-	GetTd(hash common.Hash, number uint64) *big.Int
 }
 
 // ChainReader defines a small collection of methods needed to access the local

core/bench_test.go

@@ -285,7 +285,6 @@ func makeChainForBench(db ethdb.Database, genesis *Genesis, full bool, count uin
 
 		rawdb.WriteHeader(db, header)
 		rawdb.WriteCanonicalHash(db, hash, n)
-		rawdb.WriteTd(db, hash, n, big.NewInt(int64(n+1)))
 
 		if n == 0 {
 			rawdb.WriteChainConfig(db, hash, genesis.Config)