Less broken things

All tests pass at the expense of cardboarded and duck-taped code. This needs some serious refactoring, perhaps more tests, then squashing all these WIP commits.

eth_tester/backends/pyevm/main.py

@@ -8,7 +8,6 @@
 from eth_abi.exceptions import (
     DecodingError
 )
-from toolz import dissoc
 
 from eth_utils import (
     encode_hex,
@@ -120,6 +119,7 @@ def generate_genesis_state_for_keys(account_keys, overrides=None):
 
 
 def get_default_genesis_params(overrides=None):
+    # commented out params became un-configurable in London
     default_genesis_params = {
         # "bloom": 0,
         "coinbase": GENESIS_COINBASE,
@@ -451,6 +451,11 @@ def _normalize_transaction(self, transaction, block_number='latest'):
             yield 'value', 0
         if 'to' not in transaction:
             yield 'to', b''
+        if (
+            all(_ in transaction for _ in ('max_fee_per_gas', 'max_priority_fee_per_gas')) and
+            'access_list' not in transaction
+        ):
+            yield 'access_list', []
 
     def _get_normalized_and_unsigned_evm_transaction(self, transaction, block_number='latest'):
         normalized_transaction = self._normalize_transaction(transaction, block_number)
@@ -470,14 +475,10 @@ def _get_normalized_and_signed_evm_transaction(self, transaction, block_number='
 
     def _create_type_aware_unsigned_transaction(self, normalized_txn):
         if all(_ in normalized_txn for _ in ("access_list", "gas_price")):
-            # validated type='0x1' so can pop it back out since type is signed separate from txn
-            normalized_txn = dissoc(normalized_txn, 'type')
             return self.chain.get_transaction_builder().new_unsigned_access_list_transaction(
                 **normalized_txn
             )
         elif all(_ in normalized_txn for _ in ("max_fee_per_gas", "max_priority_fee_per_gas")):
-            # validated type='0x2' so can pop it back out since type is signed separate from txn
-            normalized_txn = dissoc(normalized_txn, 'type')
             return self.chain.get_transaction_builder().new_unsigned_dynamic_fee_transaction(
                 **normalized_txn
             )

eth_tester/backends/pyevm/serializers.py

@@ -1,6 +1,10 @@
 import rlp
-from toolz import dissoc, merge
 
+from toolz import (
+    merge,
+)
+
+from eth_tester.exceptions import ValidationError
 from eth_tester.utils.address import (
     generate_contract_address,
 )
@@ -48,6 +52,7 @@ def serialize_block(block, full_transaction, is_pending):
         "timestamp": block.header.timestamp,
         "transactions": transactions,
         "uncles": [uncle.hash for uncle in block.uncles],
+        "base_fee_per_gas": block.header.base_fee_per_gas,
     }
 
 
@@ -67,44 +72,57 @@ def serialize_transaction(block, transaction, transaction_index, is_pending):
         "value": transaction.value,
         "gas": transaction.gas,
         "data": transaction.data,
+        "r": transaction.r,
+        "s": transaction.s,
     }
-    if hasattr(transaction, 'gas_price'):
-        if hasattr(transaction, 'access_list'):
+    if _field_in_transaction(transaction, 'gas_price'):
+        if _field_in_transaction(transaction, 'access_list'):
             # access list transaction
-            serialized_transaction = merge(
-                common_transaction_params,
-                {
-                    'gas_price': transaction.gas_price,
-                    'access_list': transaction.access_list,
-                }
-            )
+            type_specific_params = {
+                'chain_id': transaction.chain_id,
+                'gas_price': transaction.gas_price,
+                'access_list': transaction.access_list or [],
+                'y_parity': transaction.y_parity,
+            }
         else:
             # legacy transaction
-            serialized_transaction = merge(
-                common_transaction_params,
-                {
-                    "gas_price": transaction.gas_price,
-                    "v": transaction.v,
-                    "r": transaction.r,
-                    "s": transaction.s,
-                }
-            )
-    elif (
-        hasattr(transaction, 'max_fee_per_gas') and
-        hasattr(transaction, 'max_priority_fee_per_gas')
-    ):
-        # dynamic fee transaction
-        serialized_transaction = merge(
-            common_transaction_params,
-            {
-                'max_fee_per_gas': transaction.max_fee_per_gas,
-                'max_priority_fee_per_gas': transaction.max_priority_fee_per_gas,
-                'access_list': transaction.access_list or None,
+            type_specific_params = {
+                'gas_price': transaction.gas_price,
+                'v': transaction.v
             }
-        )
+    elif (_field_in_transaction(transaction, _) for _ in (
+        'max_fee_per_gas' and 'max_priority_fee_per_gas'
+    )):
+        # dynamic fee transaction
+        type_specific_params = {
+            'chain_id': transaction.chain_id,
+            'max_fee_per_gas': transaction.max_fee_per_gas,
+            'max_priority_fee_per_gas': transaction.max_priority_fee_per_gas,
+            'access_list': transaction.access_list or [],
+            'y_parity': transaction.y_parity,
+        }
     else:
-        raise NotImplementedError('transaction type not recognized for serialization')
-    return serialized_transaction
+        raise ValidationError('Transaction serialization error')
+    return merge(common_transaction_params, type_specific_params)
+
+
+def _field_in_transaction(transaction, field):
+    """
+    There are many different classes of transactions, we have to be able to search for a
+    particular field depending on the type of transaction - from a dict, to *LegacyTransaction to
+    *TypedTransaction.
+    """
+    if not isinstance(transaction, dict):
+        try:
+            txn_dict = transaction.as_dict()
+            if not isinstance(txn_dict, dict):
+                txn_dict = txn_dict.dictionary
+        except AttributeError:
+            pass
+    try:
+        return field in txn_dict
+    except (NameError, TypeError):
+        return hasattr(transaction, field)
 
 
 def serialize_transaction_receipt(

eth_tester/normalization/inbound.py

@@ -1,6 +1,10 @@
 from __future__ import absolute_import
 
-from eth_utils import encode_hex
+from eth_utils import (
+    encode_hex,
+    remove_0x_prefix,
+    to_bytes,
+)
 
 from eth_utils.curried import (
     apply_one_of_formatters,
@@ -83,6 +87,19 @@ def normalize_private_key(value):
     (is_hex, to_canonical_address),
 ))
 
+
+def _normalize_inbound_access_list(access_list):
+    normalized_access_list = []
+    for entry in access_list:
+        normalized_address = to_bytes(hexstr=entry.get('address'))
+        normalized_storage_keys = []
+        for k in entry.get('storage_keys'):
+            stripped_0x = remove_0x_prefix(k)
+            normalized_storage_keys.append(int(stripped_0x))
+        normalized_access_list.append((normalized_address, tuple(normalized_storage_keys)))
+    return tuple(normalized_access_list)
+
+
 TRANSACTION_NORMALIZERS = {
     'chain_id': identity,
     'type': encode_hex,
@@ -95,13 +112,12 @@ def normalize_private_key(value):
     'nonce': identity,
     'value': identity,
     'data': decode_hex,
-    'access_list': identity,
+    'access_list': _normalize_inbound_access_list,
     'r': identity,
     's': identity,
     'v': identity,
+    'y_parity': identity,
 }
-
-
 normalize_transaction = partial(normalize_dict, normalizers=TRANSACTION_NORMALIZERS)
 
 
@@ -116,8 +132,6 @@ def normalize_private_key(value):
     'data': decode_hex,
     'topics': partial(normalize_array, normalizer=decode_hex),
 }
-
-
 normalize_log_entry = partial(normalize_dict, normalizers=LOG_ENTRY_NORMALIZERS)
 
 

eth_tester/normalization/outbound.py

@@ -1,7 +1,5 @@
 from __future__ import absolute_import
 
-from eth_utils import is_integer
-
 from eth_utils.curried import (
     apply_one_of_formatters,
     to_checksum_address,
@@ -23,6 +21,7 @@
     normalize_dict,
     normalize_array,
 )
+from ..utils.encoding import int_to_32byte_big_endian
 
 
 normalize_account = to_checksum_address
@@ -33,9 +32,23 @@
     (is_canonical_address, to_checksum_address),
 ))
 
+
+def normalize_access_list(access_list):
+    formatted_access_list = []
+    for entry in access_list:
+        account = to_checksum_address(entry['account'])
+        storage_keys = tuple(
+            [encode_hex(int_to_32byte_big_endian(k)) for k in entry['storage_keys']]
+        )
+        formatted_access_list.append({
+            'address': account,
+            'storage_keys': tuple(storage_keys)
+        })
+    return tuple(formatted_access_list)
+
+
 TRANSACTION_NORMALIZERS = {
     "chain_id": identity,
-    "type": encode_hex,
     "hash": encode_hex,
     "nonce": identity,
     "block_hash": partial(normalize_if, conditional_fn=is_bytes, normalizer=encode_hex),
@@ -49,10 +62,11 @@
     "max_fee_per_gas": identity,
     "max_priority_fee_per_gas": identity,
     "data": encode_hex,
-    "access_list": identity,
-    "v": identity,
+    "access_list": normalize_access_list,
     "r": identity,
     "s": identity,
+    "v": identity,
+    "y_parity": identity,
 }
 normalize_transaction = partial(normalize_dict, normalizers=TRANSACTION_NORMALIZERS)
 

eth_tester/utils/backend_testing.py

@@ -322,25 +322,68 @@ def test_send_transaction(self, eth_tester, test_transaction):
 
         self._send_and_check_transaction(eth_tester, test_transaction, accounts[0])
 
-    def test_outbound_transaction_formatters(self, eth_tester):
+    def test_send_access_list_transaction(self, eth_tester):
         accounts = eth_tester.get_accounts()
         assert accounts, "No accounts available for transaction sending"
 
-        test_transaction = {
+        access_list_transaction = {
+            'chain_id': 123456789,
+            'from': accounts[0],
+            'to': accounts[0],
+            'value': 1,
+            'gas': 40000,
+            'gas_price': 1000000000,
+            'access_list': [],
+        }
+        self._send_and_check_transaction(eth_tester, access_list_transaction, accounts[0])
+
+        # with access list
+        access_list_transaction['access_list'] = (
+            {
+                'address': '0xde0b295669a9fd93d5f28d9ec85e40f4cb697bae',
+                'storage_keys': (
+                    '0x0000000000000000000000000000000000000000000000000000000000000003',
+                    '0x0000000000000000000000000000000000000000000000000000000000000007',
+                )
+            },
+            {
+                'address': '0xbb9bc244d798123fde783fcc1c72d3bb8c189413',
+                'storage_keys': ()
+            },
+        )
+        self._send_and_check_transaction(eth_tester, access_list_transaction, accounts[0])
+
+    def test_send_dynamic_fee_transaction(self, eth_tester):
+        accounts = eth_tester.get_accounts()
+        assert accounts, "No accounts available for transaction sending"
+
+        dynamic_fee_transaction = {
             'chain_id': 123456789,
-            'type': '0x02',
             'from': accounts[0],
             'to': accounts[0],
             'value': 1,
             'gas': 40000,
             'max_fee_per_gas': 2000000000,
             'max_priority_fee_per_gas': 1000000000,
-            'access_list': ((b'\xf0' * 20, (2, 3)),),
+            'access_list': [],
         }
-
-        txn_hash = eth_tester.send_transaction(test_transaction)
-        txn = eth_tester.get_transaction_by_hash(txn_hash)
-        assert txn
+        self._send_and_check_transaction(eth_tester, dynamic_fee_transaction, accounts[0])
+
+        # with access list
+        dynamic_fee_transaction['access_list'] = (
+            {
+                'address': '0xde0b295669a9fd93d5f28d9ec85e40f4cb697bae',
+                'storage_keys': (
+                    '0x0000000000000000000000000000000000000000000000000000000000000003',
+                    '0x0000000000000000000000000000000000000000000000000000000000000007',
+                )
+            },
+            {
+                'address': '0xbb9bc244d798123fde783fcc1c72d3bb8c189413',
+                'storage_keys': ()
+            },
+        )
+        self._send_and_check_transaction(eth_tester, dynamic_fee_transaction, accounts[0])
 
     def test_block_number_auto_mine_transactions_enabled(self, eth_tester):
         eth_tester.mine_blocks()

eth_tester/validation/common.py

@@ -129,9 +129,7 @@ def validate_transaction_params(value):
     )):
         raise ValidationError("legacy gas price and dynamic fee transaction parameters present")
     if "type" in value:
-        if value["type"] in ("0x1", 1) and "access_list" not in value:
-            raise ValidationError("type 1 transaction is missing access_list")
-        if value["type"] in ("0x2", 2) and not all(_ in value for _ in (
+        if value["type"] == "0x2" and not (_ in value for _ in (
             "max_fee_per_gas", "max_priority_fee_per_gas"
         )):
             raise ValidationError("type 2 transaction is missing fee values")
@@ -154,7 +152,6 @@ def validate_dict(value, key_validators):
         validate_transaction_params(value)  # if transaction
     else:
         validate_has_required_keys(value, key_validators.keys())
-
     key_errors = _accumulate_dict_errors(value, key_validators)
     if key_errors:
         key_messages = tuple(