blockchian: support to import block with blob & blobGasFee; (#2260)

* blob: implement min&max gas price logic;
blob: change some key parameters;

* blockchian: support import side chain;

* blobpool: reject the banned address;

* blockchain: add chasing head for DA check;

* params: update blob related config;

* blockchain: opt data available checking performance;
dataavailability: add more UTs;

* blockchain: opt data available checking performance;
dataavailability: add more UTs;

* ci: fix failed UTs;

* ci: fix failed UTs;

* params: modify blob related params;

* gasprice: support BEP-336 blob gas price calculate;

* ci: fix some broken UTs;

cmd/devp2p/internal/ethtest/suite.go

@@ -773,7 +773,7 @@ func (s *Suite) makeBlobTxs(count, blobs int, discriminator byte) (txs types.Tra
 			GasTipCap:  uint256.NewInt(1),
 			GasFeeCap:  uint256.MustFromBig(s.chain.Head().BaseFee()),
 			Gas:        100000,
-			BlobFeeCap: uint256.MustFromBig(eip4844.CalcBlobFee(*s.chain.Head().ExcessBlobGas())),
+			BlobFeeCap: uint256.MustFromBig(eip4844.CalcBlobFee(*s.chain.Head().ExcessBlobGas(), nil)),
 			BlobHashes: makeSidecar(blobdata...).BlobHashes(),
 			Sidecar:    makeSidecar(blobdata...),
 		}

cmd/evm/internal/t8ntool/execution.go

@@ -169,15 +169,15 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig,
 	var excessBlobGas uint64
 	if pre.Env.ExcessBlobGas != nil {
 		excessBlobGas := *pre.Env.ExcessBlobGas
-		vmContext.BlobBaseFee = eip4844.CalcBlobFee(excessBlobGas)
+		vmContext.BlobBaseFee = eip4844.CalcBlobFee(excessBlobGas, chainConfig)
 	} else {
 		// If it is not explicitly defined, but we have the parent values, we try
 		// to calculate it ourselves.
 		parentExcessBlobGas := pre.Env.ParentExcessBlobGas
 		parentBlobGasUsed := pre.Env.ParentBlobGasUsed
 		if parentExcessBlobGas != nil && parentBlobGasUsed != nil {
 			excessBlobGas = eip4844.CalcExcessBlobGas(*parentExcessBlobGas, *parentBlobGasUsed)
-			vmContext.BlobBaseFee = eip4844.CalcBlobFee(excessBlobGas)
+			vmContext.BlobBaseFee = eip4844.CalcBlobFee(excessBlobGas, chainConfig)
 		}
 	}
 	// If DAO is supported/enabled, we need to handle it here. In geth 'proper', it's

cmd/utils/flags.go

@@ -1096,10 +1096,10 @@ Please note that --` + MetricsHTTPFlag.Name + ` must be set to start the server.
 	}
 
 	// Blob setting
-	BlobExtraReserveFlag = &cli.Int64Flag{
+	BlobExtraReserveFlag = &cli.Uint64Flag{
 		Name:  "blob.extra-reserve",
-		Usage: "Extra reserve threshold for blob, blob never expires when -1 is set, default 28800",
-		Value: params.BlobExtraReserveThreshold,
+		Usage: "Extra reserve threshold for blob, blob never expires when 0 is set, default 28800",
+		Value: params.DefaultExtraReserveForBlobRequests,
 	}
 )
 
@@ -2122,7 +2122,11 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 
 	// blob setting
 	if ctx.IsSet(BlobExtraReserveFlag.Name) {
-		cfg.BlobExtraReserve = ctx.Int64(BlobExtraReserveFlag.Name)
+		extraReserve := ctx.Uint64(BlobExtraReserveFlag.Name)
+		if extraReserve > 0 && extraReserve < params.DefaultExtraReserveForBlobRequests {
+			extraReserve = params.DefaultExtraReserveForBlobRequests
+		}
+		cfg.BlobExtraReserve = extraReserve
 	}
 }
 

consensus/consensus.go

@@ -58,6 +58,9 @@ type ChainHeaderReader interface {
 
 	// GetHighestVerifiedHeader retrieves the highest header verified.
 	GetHighestVerifiedHeader() *types.Header
+
+	// ChasingHead return the best chain head of peers.
+	ChasingHead() *types.Header
 }
 
 type VotePool interface {
@@ -157,5 +160,4 @@ type PoSA interface {
 	GetFinalizedHeader(chain ChainHeaderReader, header *types.Header) *types.Header
 	VerifyVote(chain ChainHeaderReader, vote *types.VoteEnvelope) error
 	IsActiveValidatorAt(chain ChainHeaderReader, header *types.Header, checkVoteKeyFn func(bLSPublicKey *types.BLSPublicKey) bool) bool
-	IsDataAvailable(chain ChainHeaderReader, block *types.Block) error
 }

consensus/misc/eip4844/eip4844.go

@@ -76,8 +76,19 @@ func CalcExcessBlobGas(parentExcessBlobGas uint64, parentBlobGasUsed uint64) uin
 }
 
 // CalcBlobFee calculates the blobfee from the header's excess blob gas field.
-func CalcBlobFee(excessBlobGas uint64) *big.Int {
-	return fakeExponential(minBlobGasPrice, new(big.Int).SetUint64(excessBlobGas), blobGaspriceUpdateFraction)
+// config param aims to support eth's test suit.
+func CalcBlobFee(excessBlobGas uint64, config *params.ChainConfig) *big.Int {
+	dynamicPrice := fakeExponential(minBlobGasPrice, new(big.Int).SetUint64(excessBlobGas), blobGaspriceUpdateFraction)
+	if config == nil || config.Parlia == nil {
+		return dynamicPrice
+	}
+	if dynamicPrice.Cmp(params.MinBlobGasPriceInBSC) < 0 {
+		return params.MinBlobGasPriceInBSC
+	}
+	if dynamicPrice.Cmp(params.MaxBlobGasPriceInBSC) > 0 {
+		return params.MaxBlobGasPriceInBSC
+	}
+	return dynamicPrice
 }
 
 // fakeExponential approximates factor * e ** (numerator / denominator) using

consensus/misc/eip4844/eip4844_test.go

@@ -61,14 +61,20 @@ func TestCalcBlobFee(t *testing.T) {
 	tests := []struct {
 		excessBlobGas uint64
 		blobfee       int64
+		config        *params.ChainConfig
 	}{
-		{0, 1},
-		{2314057, 1},
-		{2314058, 2},
-		{10 * 1024 * 1024, 23},
+		{0, 1, nil},
+		{2314057, 1, nil},
+		{2314058, 2, nil},
+		{10 * 1024 * 1024, 23, nil},
+		{0, params.MinBlobGasPriceInBSC.Int64(), params.ParliaTestChainConfig},
+		{555 * params.BlobTxBlobGasPerBlob, params.MinBlobGasPriceInBSC.Int64(), params.ParliaTestChainConfig},
+		{556 * params.BlobTxBlobGasPerBlob, 3021819819, params.ParliaTestChainConfig},
+		{627 * params.BlobTxBlobGasPerBlob, 49077044416, params.ParliaTestChainConfig},
+		{628 * params.BlobTxBlobGasPerBlob, params.MaxBlobGasPriceInBSC.Int64(), params.ParliaTestChainConfig},
 	}
 	for i, tt := range tests {
-		have := CalcBlobFee(tt.excessBlobGas)
+		have := CalcBlobFee(tt.excessBlobGas, tt.config)
 		if have.Int64() != tt.blobfee {
 			t.Errorf("test %d: blobfee mismatch: have %v want %v", i, have, tt.blobfee)
 		}

consensus/parlia/parlia.go

@@ -350,46 +350,6 @@ func (p *Parlia) VerifyHeaders(chain consensus.ChainHeaderReader, headers []*typ
 	return abort, results
 }
 
-// IsDataAvailable it checks that the blobTx block has available blob data
-func (p *Parlia) IsDataAvailable(chain consensus.ChainHeaderReader, block *types.Block) error {
-	if !p.chainConfig.IsCancun(block.Number(), block.Time()) {
-		return nil
-	}
-	// only required to check within BlobReserveThreshold block's DA
-	currentHeader := chain.CurrentHeader()
-	if block.NumberU64() < currentHeader.Number.Uint64()-params.BlobReserveThreshold {
-		return nil
-	}
-
-	// alloc block's versionedHashes
-	versionedHashes := make([][]common.Hash, 0, len(block.Transactions()))
-	for _, tx := range block.Transactions() {
-		versionedHashes = append(versionedHashes, tx.BlobHashes())
-	}
-	blobs := block.Blobs()
-	if len(versionedHashes) != len(blobs) {
-		return errors.New("blobs do not match the versionedHashes length")
-	}
-
-	// check blob amount
-	blobCnt := 0
-	for _, h := range versionedHashes {
-		blobCnt += len(h)
-	}
-	if blobCnt > params.MaxBlobGasPerBlock/params.BlobTxBlobGasPerBlob {
-		return fmt.Errorf("too many blobs in block: have %d, permitted %d", blobCnt, params.MaxBlobGasPerBlock/params.BlobTxBlobGasPerBlob)
-	}
-
-	// check blob and versioned hash
-	for i := range versionedHashes {
-		if err := validateBlobSidecar(versionedHashes[i], blobs[i]); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
 // getValidatorBytesFromHeader returns the validators bytes extracted from the header's extra field if exists.
 // The validators bytes would be contained only in the epoch block's header, and its each validator bytes length is fixed.
 // On luban fork, we introduce vote attestation into the header's extra field, so extra format is different from before.
@@ -2002,6 +1962,10 @@ func (c chainContext) Engine() consensus.Engine {
 	return c.parlia
 }
 
+func (c chainContext) Config() *params.ChainConfig {
+	return c.Chain.Config()
+}
+
 func (c chainContext) GetHeader(hash common.Hash, number uint64) *types.Header {
 	return c.Chain.GetHeader(hash, number)
 }

core/blockchain.go

@@ -272,6 +272,7 @@ type BlockChain struct {
 	highestVerifiedHeader atomic.Pointer[types.Header]
 	currentBlock          atomic.Pointer[types.Header] // Current head of the chain
 	currentSnapBlock      atomic.Pointer[types.Header] // Current head of snap-sync
+	chasingHead           atomic.Pointer[types.Header]
 
 	bodyCache     *lru.Cache[common.Hash, *types.Body]
 	bodyRLPCache  *lru.Cache[common.Hash, rlp.RawValue]
@@ -392,6 +393,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis
 	bc.highestVerifiedHeader.Store(nil)
 	bc.currentBlock.Store(nil)
 	bc.currentSnapBlock.Store(nil)
+	bc.chasingHead.Store(nil)
 
 	// Update chain info data metrics
 	chainInfoGauge.Update(metrics.GaugeInfoValue{"chain_id": bc.chainConfig.ChainID.String()})
@@ -648,6 +650,9 @@ func (bc *BlockChain) cacheDiffLayer(diffLayer *types.DiffLayer, diffLayerCh cha
 
 func (bc *BlockChain) cacheBlock(hash common.Hash, block *types.Block) {
 	bc.blockCache.Add(hash, block)
+	if bc.chainConfig.IsCancun(block.Number(), block.Time()) {
+		bc.blobsCache.Add(hash, block.Blobs())
+	}
 }
 
 // empty returns an indicator whether the blockchain is empty.
@@ -1037,6 +1042,16 @@ func (bc *BlockChain) SnapSyncCommitHead(hash common.Hash) error {
 	return nil
 }
 
+// UpdateChasingHead update remote best chain head, used by DA check now.
+func (bc *BlockChain) UpdateChasingHead(head *types.Header) {
+	bc.chasingHead.Store(head)
+}
+
+// ChasingHead return the best chain head of peers.
+func (bc *BlockChain) ChasingHead() *types.Header {
+	return bc.chasingHead.Load()
+}
+
 // Reset purges the entire blockchain, restoring it to its genesis state.
 func (bc *BlockChain) Reset() error {
 	return bc.ResetWithGenesisBlock(bc.genesisBlock)
@@ -1380,8 +1395,9 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
 
 		// Write all chain data to ancients.
 		td := bc.GetTd(first.Hash(), first.NumberU64())
-		// TODO(GalaIO): when sync the history block, it needs store blobs too.
+		// TODO(GalaIO): when sync the history block, it needs check DA & store blobs too.
 		//if isCancun() {
+		//  posa.IsDataAvailable()
 		//	writeSize, err := rawdb.WriteAncientBlocksWithBlobs(bc.db, blockChain, receiptChain, td, blobs)
 		//}
 		writeSize, err := rawdb.WriteAncientBlocks(bc.db, blockChain, receiptChain, td)
@@ -1462,8 +1478,9 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
 			// Write all the data out into the database
 			rawdb.WriteBody(batch, block.Hash(), block.NumberU64(), block.Body())
 			rawdb.WriteReceipts(batch, block.Hash(), block.NumberU64(), receiptChain[i])
-			// TODO(GalaIO): if enable cancun, need write blobs
+			// TODO(GalaIO): if enable cancun, need check DA & write blobs
 			//if bc.chainConfig.IsCancun(block.Number(), block.Time()) {
+			//  posa.IsDataAvailable()
 			//	rawdb.WriteBlobs(batch, block.Hash(), block.NumberU64(), blobs)
 			//}
 
@@ -1535,7 +1552,7 @@ func (bc *BlockChain) writeBlockWithoutState(block *types.Block, td *big.Int) (e
 	batch := bc.db.NewBatch()
 	rawdb.WriteTd(batch, block.Hash(), block.NumberU64(), td)
 	rawdb.WriteBlock(batch, block)
-	// if enable cancun, it needs to write blobs too
+	// if cancun is enabled, here need to write blobs too
 	if bc.chainConfig.IsCancun(block.Number(), block.Time()) {
 		rawdb.WriteBlobs(batch, block.Hash(), block.NumberU64(), block.Blobs())
 	}
@@ -1581,7 +1598,7 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.
 		rawdb.WriteTd(blockBatch, block.Hash(), block.NumberU64(), externTd)
 		rawdb.WriteBlock(blockBatch, block)
 		rawdb.WriteReceipts(blockBatch, block.Hash(), block.NumberU64(), receipts)
-		// if enable cancun, it needs to write blobs too
+		// if cancun is enabled, here need to write blobs too
 		if bc.chainConfig.IsCancun(block.Number(), block.Time()) {
 			rawdb.WriteBlobs(blockBatch, block.Hash(), block.NumberU64(), block.Blobs())
 		}
@@ -1822,7 +1839,6 @@ func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) {
 // racey behaviour. If a sidechain import is in progress, and the historic state
 // is imported, but then new canon-head is added before the actual sidechain
 // completes, then the historic state could be pruned again
-// TODO(GalaIO): if enable cancun, it must set received blob cache for check, remove cache when failed
 func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error) {
 	// If the chain is terminating, don't even bother starting up.
 	if bc.insertStopped() {
@@ -1848,6 +1864,14 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error)
 			}
 		}
 	}()
+
+	// check block data available first
+	if bc.chainConfig.Parlia != nil {
+		if index, err := CheckDataAvailableInBatch(bc, chain); err != nil {
+			return index, err
+		}
+	}
+
 	// Start the parallel header verifier
 	headers := make([]*types.Header, len(chain))
 	for i, block := range chain {
@@ -2006,15 +2030,6 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error)
 			parent = bc.GetHeader(block.ParentHash(), block.NumberU64()-1)
 		}
 
-		// check blob data available first
-		// TODO(GalaIO): move IsDataAvailable combine into verifyHeaders?
-		if bc.chainConfig.IsCancun(block.Number(), block.Time()) {
-			if posa, ok := bc.engine.(consensus.PoSA); ok {
-				if err = posa.IsDataAvailable(bc, block); err != nil {
-					return it.index, err
-				}
-			}
-		}
 		statedb, err := state.NewWithSharedPool(parent.Root, bc.stateCache, bc.snaps)
 		if err != nil {
 			return it.index, err
@@ -2316,7 +2331,9 @@ func (bc *BlockChain) insertSideChain(block *types.Block, it *insertIterator) (i
 		if block == nil {
 			log.Crit("Importing heavy sidechain block is nil", "hash", hashes[i], "number", numbers[i])
 		}
-
+		if bc.chainConfig.IsCancun(block.Number(), block.Time()) {
+			block = block.WithBlobs(bc.GetBlobsByHash(hashes[i]))
+		}
 		blocks = append(blocks, block)
 		memory += block.Size()
 
@@ -2388,6 +2405,9 @@ func (bc *BlockChain) recoverAncestors(block *types.Block) (common.Hash, error)
 		} else {
 			b = bc.GetBlock(hashes[i], numbers[i])
 		}
+		if bc.chainConfig.IsCancun(b.Number(), b.Time()) {
+			b = b.WithBlobs(bc.GetBlobsByHash(b.Hash()))
+		}
 		if _, err := bc.insertChain(types.Blocks{b}, false); err != nil {
 			return b.ParentHash(), err
 		}
@@ -2401,7 +2421,7 @@ func (bc *BlockChain) collectLogs(b *types.Block, removed bool) []*types.Log {
 	var blobGasPrice *big.Int
 	excessBlobGas := b.ExcessBlobGas()
 	if excessBlobGas != nil {
-		blobGasPrice = eip4844.CalcBlobFee(*excessBlobGas)
+		blobGasPrice = eip4844.CalcBlobFee(*excessBlobGas, bc.chainConfig)
 	}
 	receipts := rawdb.ReadRawReceipts(bc.db, b.Hash(), b.NumberU64())
 	if err := receipts.DeriveFields(bc.chainConfig, b.Hash(), b.NumberU64(), b.Time(), b.BaseFee(), blobGasPrice, b.Transactions()); err != nil {
@@ -2826,6 +2846,7 @@ func (bc *BlockChain) isCachedBadBlock(block *types.Block) bool {
 }
 
 // reportBlock logs a bad block error.
+// bad block need not save receipts & blobs.
 func (bc *BlockChain) reportBlock(block *types.Block, receipts types.Receipts, err error) {
 	rawdb.WriteBadBlock(bc.db, block)
 	log.Error(summarizeBadBlock(block, receipts, bc.Config(), err))

core/blockchain_test.go

@@ -3585,10 +3585,19 @@ func TestEIP2718TransitionWithTestChainConfig(t *testing.T) {
 	testEIP2718TransitionWithConfig(t, rawdb.HashScheme, params.TestChainConfig)
 }
 
+func preShanghaiConfig() *params.ChainConfig {
+	config := *params.ParliaTestChainConfig
+	config.ShanghaiTime = nil
+	config.KeplerTime = nil
+	config.FeynmanTime = nil
+	config.CancunTime = nil
+	return &config
+}
+
 // TestEIP2718TransitionWithParliaConfig tests EIP-2718 with Parlia Config.
 func TestEIP2718TransitionWithParliaConfig(t *testing.T) {
-	testEIP2718TransitionWithConfig(t, rawdb.HashScheme, params.ParliaTestChainConfig)
-	testEIP2718TransitionWithConfig(t, rawdb.PathScheme, params.ParliaTestChainConfig)
+	testEIP2718TransitionWithConfig(t, rawdb.HashScheme, preShanghaiConfig())
+	testEIP2718TransitionWithConfig(t, rawdb.PathScheme, preShanghaiConfig())
 }
 
 // testEIP2718TransitionWithConfig tests EIP02718 with given ChainConfig.