Refactor consensus handling to be more flexible and sound

eth/abc.py

@@ -43,6 +43,7 @@
     JournalDBCheckpoint,
     AccountState,
     HeaderParams,
+    VMConfiguration,
 )
 
 
@@ -2269,6 +2270,59 @@ def get_transaction_context(cls,
         ...
 
 
+class ConsensusContextAPI(ABC):
+    """
+    A class representing a data context for the :class:`~eth.abc.ConsensusAPI` which is
+    instantiated once per chain instance and stays in memory across VM runs.
+    """
+
+    @abstractmethod
+    def __init__(self, db: AtomicDatabaseAPI) -> None:
+        """
+        Initialize the context with a database.
+        """
+        ...
+
+
+class ConsensusAPI(ABC):
+    """
+    A class encapsulating the consensus scheme to allow chains to run under different kind of
+    EVM-compatible consensus mechanisms such as the Clique Proof of Authority scheme.
+    """
+
+    @abstractmethod
+    def __init__(self, context: ConsensusContextAPI) -> None:
+        """
+        Initialize the consensus api.
+        """
+        ...
+
+    @abstractmethod
+    def validate_seal(self, header: BlockHeaderAPI) -> None:
+        """
+        Validate the seal on the given header, even if its parent is missing.
+        """
+        ...
+
+    @abstractmethod
+    def validate_seal_extension(self,
+                                header: BlockHeaderAPI,
+                                parents: Iterable[BlockHeaderAPI]) -> None:
+        """
+        Validate the seal on the given header when all parents must be present. Parent headers
+        that are not yet in the database must be passed as ``parents``.
+        """
+        ...
+
+    @classmethod
+    @abstractmethod
+    def get_fee_recipient(cls, header: BlockHeaderAPI) -> Address:
+        """
+        Return the address that should receive rewards for creating the block.
+        """
+        ...
+
+
 class VirtualMachineAPI(ConfigurableAPI):
     """
     The :class:`~eth.abc.VirtualMachineAPI` class represents the Chain rules for a
@@ -2285,9 +2339,14 @@ class VirtualMachineAPI(ConfigurableAPI):
     fork: str  # noqa: E701  # flake8 bug that's fixed in 3.6.0+
     chaindb: ChainDatabaseAPI
     extra_data_max_bytes: ClassVar[int]
+    consensus_class: Type[ConsensusAPI]
+    consensus_context: ConsensusContextAPI
 
     @abstractmethod
-    def __init__(self, header: BlockHeaderAPI, chaindb: ChainDatabaseAPI) -> None:
+    def __init__(self,
+                 header: BlockHeaderAPI,
+                 chaindb: ChainDatabaseAPI,
+                 consensus_context: ConsensusContextAPI) -> None:
         """
         Initialize the virtual machine.
         """
@@ -2637,9 +2696,8 @@ def validate_block(self, block: BlockAPI) -> None:
         """
         ...
 
-    @classmethod
     @abstractmethod
-    def validate_header(cls,
+    def validate_header(self,
                         header: BlockHeaderAPI,
                         parent_header: BlockHeaderAPI,
                         check_seal: bool = True
@@ -2663,14 +2721,23 @@ def validate_transaction_against_header(self,
         """
         ...
 
-    @classmethod
     @abstractmethod
-    def validate_seal(cls, header: BlockHeaderAPI) -> None:
+    def validate_seal(self, header: BlockHeaderAPI) -> None:
         """
         Validate the seal on the given header.
         """
         ...
 
+    @abstractmethod
+    def validate_seal_extension(self,
+                                header: BlockHeaderAPI,
+                                parents: Iterable[BlockHeaderAPI]) -> None:
+        """
+        Validate the seal on the given header when all parents must be present. Parent headers
+        that are not yet in the database must be passed as ``parents``.
+        """
+        ...
+
     @classmethod
     @abstractmethod
     def validate_uncle(cls,
@@ -2703,6 +2770,20 @@ def state_in_temp_block(self) -> ContextManager[StateAPI]:
         ...
 
 
+class VirtualMachineModifierAPI(ABC):
+    """
+    Amend a set of VMs for a chain. This allows modifying a chain for different consensus schemes.
+    """
+
+    @abstractmethod
+    def amend_vm_configuration(self, vm_config: VMConfiguration) -> VMConfiguration:
+        """
+        Amend the ``vm_config`` by configuring the VM classes, and hence returning a modified
+        set of VM classes.
+        """
+        ...
+
+
 class HeaderChainAPI(ABC):
     """
     Like :class:`eth.abc.ChainAPI` but does only support headers, not entire blocks.
@@ -2814,6 +2895,7 @@ class ChainAPI(ConfigurableAPI):
     vm_configuration: Tuple[Tuple[BlockNumber, Type[VirtualMachineAPI]], ...]
     chain_id: int
     chaindb: ChainDatabaseAPI
+    consensus_context_class: Type[ConsensusContextAPI]
 
     #
     # Helpers
@@ -3155,10 +3237,9 @@ def validate_uncles(self, block: BlockAPI) -> None:
         """
         ...
 
-    @classmethod
     @abstractmethod
     def validate_chain(
-            cls,
+            self,
             root: BlockHeaderAPI,
             descendants: Tuple[BlockHeaderAPI, ...],
             seal_check_random_sample_rate: int = 1) -> None:
@@ -3171,6 +3252,17 @@ def validate_chain(
         """
         ...
 
+    @abstractmethod
+    def validate_chain_extension(self, headers: Tuple[BlockHeaderAPI, ...]) -> None:
+        """
+        Validate a chain of headers under the assumption that the entire chain of headers is
+        present. Headers that are not already in the database must exist in ``headers``. Calling
+        this API is not a replacement for calling :meth:`~eth.abc.ChainAPI.validate_chain`, it is
+        an additional API to call at a different stage of header processing to enable consensus
+        schemes where the consensus can not be verified out of order.
+        """
+        ...
+
 
 class MiningChainAPI(ChainAPI):
     """

eth/chains/base.py

@@ -44,13 +44,17 @@
     BlockHeaderAPI,
     ChainAPI,
     ChainDatabaseAPI,
+    ConsensusContextAPI,
     VirtualMachineAPI,
     ReceiptAPI,
     ComputationAPI,
     StateAPI,
     SignedTransactionAPI,
     UnsignedTransactionAPI,
 )
+from eth.consensus import (
+    ConsensusContext,
+)
 from eth.constants import (
     EMPTY_UNCLE_HASH,
     MAX_UNCLE_DEPTH,
@@ -111,6 +115,7 @@ class BaseChain(Configurable, ChainAPI):
     """
     chaindb: ChainDatabaseAPI = None
     chaindb_class: Type[ChainDatabaseAPI] = None
+    consensus_context_class: Type[ConsensusContextAPI] = None
     vm_configuration: Tuple[Tuple[BlockNumber, Type[VirtualMachineAPI]], ...] = None
     chain_id: int = None
 
@@ -133,9 +138,8 @@ def get_vm_class(cls, header: BlockHeaderAPI) -> Type[VirtualMachineAPI]:
     #
     # Validation API
     #
-    @classmethod
     def validate_chain(
-            cls,
+            self,
             root: BlockHeaderAPI,
             descendants: Tuple[BlockHeaderAPI, ...],
             seal_check_random_sample_rate: int = 1) -> None:
@@ -158,20 +162,29 @@ def validate_chain(
                     f" but expected {encode_hex(parent.hash)}"
                 )
             should_check_seal = index in indices_to_check_seal
-            vm_class = cls.get_vm_class_for_block_number(child.block_number)
+            vm = self.get_vm(child)
             try:
-                vm_class.validate_header(child, parent, check_seal=should_check_seal)
+                vm.validate_header(child, parent, check_seal=should_check_seal)
             except ValidationError as exc:
                 raise ValidationError(
                     f"{child} is not a valid child of {parent}: {exc}"
                 ) from exc
 
+    def validate_chain_extension(self, headers: Tuple[BlockHeaderAPI, ...]) -> None:
+        for index, header in enumerate(headers):
+            vm = self.get_vm(header)
+
+            # pass in any parents that are not already in the database
+            parents = headers[:index]
+            vm.validate_seal_extension(header, parents)
+
 
 class Chain(BaseChain):
     logger = logging.getLogger("eth.chain.chain.Chain")
     gas_estimator: StaticMethod[Callable[[StateAPI, SignedTransactionAPI], int]] = None
 
     chaindb_class: Type[ChainDatabaseAPI] = ChainDB
+    consensus_context_class: Type[ConsensusContextAPI] = ConsensusContext
 
     def __init__(self, base_db: AtomicDatabaseAPI) -> None:
         if not self.vm_configuration:
@@ -182,6 +195,7 @@ def __init__(self, base_db: AtomicDatabaseAPI) -> None:
             validate_vm_configuration(self.vm_configuration)
 
         self.chaindb = self.get_chaindb_class()(base_db)
+        self.consensus_context = self.consensus_context_class(self.chaindb.db)
         self.headerdb = HeaderDB(base_db)
         if self.gas_estimator is None:
             self.gas_estimator = get_gas_estimator()
@@ -247,7 +261,13 @@ def get_vm(self, at_header: BlockHeaderAPI = None) -> VirtualMachineAPI:
         header = self.ensure_header(at_header)
         vm_class = self.get_vm_class_for_block_number(header.block_number)
         chain_context = ChainContext(self.chain_id)
-        return vm_class(header=header, chaindb=self.chaindb, chain_context=chain_context)
+
+        return vm_class(
+            header=header,
+            chaindb=self.chaindb,
+            chain_context=chain_context,
+            consensus_context=self.consensus_context
+        )
 
     #
     # Header API
@@ -484,15 +504,16 @@ def validate_receipt(self, receipt: ReceiptAPI, at_header: BlockHeaderAPI) -> No
     def validate_block(self, block: BlockAPI) -> None:
         if block.is_genesis:
             raise ValidationError("Cannot validate genesis block this way")
-        VM_class = self.get_vm_class_for_block_number(BlockNumber(block.number))
+        vm = self.get_vm(block.header)
         parent_header = self.get_block_header_by_hash(block.header.parent_hash)
-        VM_class.validate_header(block.header, parent_header, check_seal=True)
+        vm.validate_header(block.header, parent_header, check_seal=True)
+        vm.validate_seal_extension(block.header, ())
         self.validate_uncles(block)
         self.validate_gaslimit(block.header)
 
     def validate_seal(self, header: BlockHeaderAPI) -> None:
-        VM_class = self.get_vm_class_for_block_number(BlockNumber(header.block_number))
-        VM_class.validate_seal(header)
+        vm = self.get_vm(header)
+        vm.validate_seal(header)
 
     def validate_gaslimit(self, header: BlockHeaderAPI) -> None:
         parent_header = self.get_block_header_by_hash(header.parent_hash)

eth/chains/mainnet/__init__.py

@@ -47,26 +47,25 @@
 class MainnetDAOValidatorVM(HomesteadVM):
     """Only on mainnet, TheDAO fork is accompanied by special extra data. Validate those headers"""
 
-    @classmethod
-    def validate_header(cls,
+    def validate_header(self,
                         header: BlockHeaderAPI,
                         previous_header: BlockHeaderAPI,
                         check_seal: bool=True) -> None:
 
         super().validate_header(header, previous_header, check_seal)
 
         # The special extra_data is set on the ten headers starting at the fork
-        dao_fork_at = cls.get_dao_fork_block_number()
+        dao_fork_at = self.get_dao_fork_block_number()
         extra_data_block_nums = range(dao_fork_at, dao_fork_at + 10)
 
         if header.block_number in extra_data_block_nums:
-            if cls.support_dao_fork and header.extra_data != DAO_FORK_MAINNET_EXTRA_DATA:
+            if self.support_dao_fork and header.extra_data != DAO_FORK_MAINNET_EXTRA_DATA:
                 raise ValidationError(
                     f"Block {header!r} must have extra data "
                     f"{encode_hex(DAO_FORK_MAINNET_EXTRA_DATA)} not "
                     f"{encode_hex(header.extra_data)} when supporting DAO fork"
                 )
-            elif not cls.support_dao_fork and header.extra_data == DAO_FORK_MAINNET_EXTRA_DATA:
+            elif not self.support_dao_fork and header.extra_data == DAO_FORK_MAINNET_EXTRA_DATA:
                 raise ValidationError(
                     f"Block {header!r} must not have extra data "
                     f"{encode_hex(DAO_FORK_MAINNET_EXTRA_DATA)} when declining the DAO fork"

eth/chains/tester/__init__.py

@@ -146,8 +146,8 @@ class MainnetTesterChain(BaseMainnetTesterChain):
     It exposes one additional API `configure_forks` to allow for in-flight
     configuration of fork rules.
     """
-    @classmethod
-    def validate_seal(cls, block: BlockAPI) -> None:
+
+    def validate_seal(self, block: BlockAPI) -> None:
         """
         We don't validate the proof of work seal on the tester chain.
         """

eth/consensus/__init__.py

@@ -0,0 +1,9 @@
+from .applier import ConsensusApplier  # noqa: F401
+from .clique.clique import (  # noqa: F401
+    CliqueApplier,
+    CliqueConsensus,
+    CliqueConsensusContext,
+)
+from .context import ConsensusContext  # noqa: F401
+from .noproof import NoProofConsensus  # noqa: F401
+from .pow import PowConsensus  # noqa: F401

eth/consensus/applier.py

@@ -0,0 +1,37 @@
+from typing import (
+    Iterable,
+    Type,
+)
+
+from eth_utils import (
+    to_tuple,
+)
+
+from eth.abc import (
+    ConsensusAPI,
+    VirtualMachineModifierAPI,
+    VMConfiguration,
+)
+from eth.typing import (
+    VMFork,
+)
+
+
+class ConsensusApplier(VirtualMachineModifierAPI):
+    """
+    This class is used to apply simple types of consensus engines to a series of virtual machines.
+
+    Note that this *must not* be used for Clique, which has its own modifier
+    """
+
+    def __init__(self, consensus_class: Type[ConsensusAPI]) -> None:
+        self._consensus_class = consensus_class
+
+    @to_tuple
+    def amend_vm_configuration(self, config: VMConfiguration) -> Iterable[VMFork]:
+        """
+        Amend the given ``VMConfiguration`` to operate under the rules of the pre-defined consensus
+        """
+        for pair in config:
+            block_number, vm = pair
+            yield block_number, vm.configure(consensus_class=self._consensus_class)

eth/consensus/clique/__init__.py

@@ -1,5 +1,7 @@
 from .clique import (  # noqa: F401
+    CliqueApplier,
     CliqueConsensus,
+    CliqueConsensusContext,
 )
 from .constants import (  # noqa: F401
     NONCE_AUTH,