diff --git a/tests/core/pyspec/eth2spec/debug/decode.py b/tests/core/pyspec/eth2spec/debug/decode.py index 30bfd487bd..a1a3cd9c24 100644 --- a/tests/core/pyspec/eth2spec/debug/decode.py +++ b/tests/core/pyspec/eth2spec/debug/decode.py @@ -2,7 +2,8 @@ from eth2spec.utils.ssz.ssz_impl import hash_tree_root from eth2spec.utils.ssz.ssz_typing import ( uint, Container, List, boolean, - Vector, ByteVector, ByteList, Union, View + Vector, ByteVector, ByteList, Union, View, + Profile, StableContainer, ) @@ -27,6 +28,43 @@ def decode(data: Any, typ): assert (data["hash_tree_root"][2:] == hash_tree_root(ret).hex()) return ret + elif issubclass(typ, StableContainer): + temp = {} + for field_name, field_type in typ.fields().items(): + if data[field_name] is None: + temp[field_name] = None + if field_name + "_hash_tree_root" in data: + assert (data[field_name + "_hash_tree_root"][2:] == + '00' * 32) + else: + temp[field_name] = decode(data[field_name], field_type) + if field_name + "_hash_tree_root" in data: + assert (data[field_name + "_hash_tree_root"][2:] == + hash_tree_root(temp[field_name]).hex()) + ret = typ(**temp) + if "hash_tree_root" in data: + assert (data["hash_tree_root"][2:] == + hash_tree_root(ret).hex()) + return ret + elif issubclass(typ, Profile): + temp = {} + for field_name, [field_type, is_optional] in typ.fields().items(): + if data[field_name] is None: + assert is_optional + temp[field_name] = None + if field_name + "_hash_tree_root" in data: + assert (data[field_name + "_hash_tree_root"][2:] == + '00' * 32) + else: + temp[field_name] = decode(data[field_name], field_type) + if field_name + "_hash_tree_root" in data: + assert (data[field_name + "_hash_tree_root"][2:] == + hash_tree_root(temp[field_name]).hex()) + ret = typ(**temp) + if "hash_tree_root" in data: + assert (data["hash_tree_root"][2:] == + hash_tree_root(ret).hex()) + return ret elif issubclass(typ, Union): selector = int(data["selector"]) options = typ.options() diff --git a/tests/core/pyspec/eth2spec/debug/encode.py b/tests/core/pyspec/eth2spec/debug/encode.py index d93f7cf5ef..6f80092056 100644 --- a/tests/core/pyspec/eth2spec/debug/encode.py +++ b/tests/core/pyspec/eth2spec/debug/encode.py @@ -1,7 +1,8 @@ from eth2spec.utils.ssz.ssz_impl import hash_tree_root, serialize from eth2spec.utils.ssz.ssz_typing import ( uint, boolean, - Bitlist, Bitvector, Container, Vector, List, Union + Bitlist, Bitvector, Container, Vector, List, Union, + Profile, StableContainer, ) @@ -31,6 +32,21 @@ def encode(value, include_hash_tree_roots=False): if include_hash_tree_roots: ret["hash_tree_root"] = '0x' + hash_tree_root(value).hex() return ret + elif isinstance(value, (StableContainer, Profile)): + ret = {} + for field_name in value.fields().keys(): + field_value = getattr(value, field_name) + if field_value is None: + ret[field_name] = None + if include_hash_tree_roots: + ret[field_name + "_hash_tree_root"] = '0x' + '00' * 32 + else: + ret[field_name] = encode(field_value, include_hash_tree_roots) + if include_hash_tree_roots: + ret[field_name + "_hash_tree_root"] = '0x' + hash_tree_root(field_value).hex() + if include_hash_tree_roots: + ret["hash_tree_root"] = '0x' + hash_tree_root(value).hex() + return ret elif isinstance(value, Union): inner_value = value.value() return { diff --git a/tests/core/pyspec/eth2spec/debug/random_value.py b/tests/core/pyspec/eth2spec/debug/random_value.py index ff80ee0f4e..f507bf3c33 100644 --- a/tests/core/pyspec/eth2spec/debug/random_value.py +++ b/tests/core/pyspec/eth2spec/debug/random_value.py @@ -5,7 +5,8 @@ from eth2spec.utils.ssz.ssz_typing import ( View, BasicView, uint, Container, List, boolean, - Vector, ByteVector, ByteList, Bitlist, Bitvector, Union + Vector, ByteVector, ByteList, Bitlist, Bitvector, Union, + Profile, StableContainer, ) # in bytes @@ -115,6 +116,29 @@ def get_random_ssz_object(rng: Random, get_random_ssz_object(rng, field_type, max_bytes_length, max_list_length, mode, chaos) for field_name, field_type in fields.items() }) + elif issubclass(typ, StableContainer): + fields = typ.fields() + # StableContainer + return typ(**{ + field_name: + rng.choice([ + None, + get_random_ssz_object(rng, field_type, max_bytes_length, max_list_length, mode, chaos) + ]) + for field_name, field_type in fields.items() + }) + elif issubclass(typ, Profile): + fields = typ.fields() + # Profile + return typ(**{ + field_name: + rng.choice([ + None if is_optional else get_random_ssz_object( + rng, field_type, max_bytes_length, max_list_length, mode, chaos), + get_random_ssz_object(rng, field_type, max_bytes_length, max_list_length, mode, chaos) + ]) + for field_name, [field_type, is_optional] in fields.items() + }) elif issubclass(typ, Union): options = typ.options() selector: int diff --git a/tests/core/pyspec/eth2spec/utils/ssz/ssz_typing.py b/tests/core/pyspec/eth2spec/utils/ssz/ssz_typing.py index 1f3db2fe00..054c20e034 100644 --- a/tests/core/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/tests/core/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -7,6 +7,7 @@ from remerkleable.bitfields import Bitvector, Bitlist from remerkleable.byte_arrays import ByteVector, Bytes1, Bytes4, Bytes8, Bytes32, Bytes48, Bytes96, ByteList from remerkleable.core import BasicView, View, Path +from remerkleable.stable_container import Profile, StableContainer Bytes20 = ByteVector[20] # type: ignore diff --git a/tests/generators/ssz_generic/main.py b/tests/generators/ssz_generic/main.py index 2e96ce2e87..eeb79a4c0f 100644 --- a/tests/generators/ssz_generic/main.py +++ b/tests/generators/ssz_generic/main.py @@ -6,6 +6,8 @@ import ssz_boolean import ssz_uints import ssz_container +import ssz_stablecontainer +import ssz_profile from eth2spec.test.helpers.constants import PHASE0 @@ -43,4 +45,6 @@ def cases_fn() -> Iterable[gen_typing.TestCase]: create_provider("uints", "invalid", ssz_uints.invalid_cases), create_provider("containers", "valid", ssz_container.valid_cases), create_provider("containers", "invalid", ssz_container.invalid_cases), + create_provider("stablecontainers", "valid", ssz_stablecontainer.valid_cases), + create_provider("profiles", "valid", ssz_profile.valid_cases), ]) diff --git a/tests/generators/ssz_generic/ssz_profile.py b/tests/generators/ssz_generic/ssz_profile.py new file mode 100644 index 0000000000..30651ae514 --- /dev/null +++ b/tests/generators/ssz_generic/ssz_profile.py @@ -0,0 +1,228 @@ +from ssz_test_case import invalid_test_case, valid_test_case +from eth2spec.utils.ssz.ssz_typing import View, byte, uint8, uint16, \ + uint32, uint64, List, ByteList, Vector, Bitvector, Bitlist, Profile +from eth2spec.utils.ssz.ssz_impl import serialize +from random import Random +from typing import Dict, Tuple, Sequence, Callable, Type +from eth2spec.debug.random_value import RandomizationMode, get_random_ssz_object +from ssz_stablecontainer import SingleFieldTestStableStruct, SmallTestStableStruct, FixedTestStableStruct, \ + VarTestStableStruct, ComplexTestStableStruct, BitsStableStruct + + +class SingleFieldTestProfile(Profile[SingleFieldTestStableStruct]): + A: byte + + +class SmallTestProfile1(Profile[SmallTestStableStruct]): + A: uint16 + B: uint16 + + +class SmallTestProfile2(Profile[SmallTestStableStruct]): + A: uint16 + + +class SmallTestProfile3(Profile[SmallTestStableStruct]): + B: uint16 + + +class FixedTestProfile1(Profile[FixedTestStableStruct]): + A: uint8 + B: uint64 + C: uint32 + + +class FixedTestProfile2(Profile[FixedTestStableStruct]): + A: uint8 + B: uint64 + + +class FixedTestProfile3(Profile[FixedTestStableStruct]): + A: uint8 + C: uint32 + + +class FixedTestProfile4(Profile[FixedTestStableStruct]): + C: uint32 + + +class VarTestProfile1(Profile[VarTestStableStruct]): + A: uint16 + B: List[uint16, 1024] + C: uint8 + + +class VarTestProfile2(Profile[VarTestStableStruct]): + B: List[uint16, 1024] + C: uint8 + + +class VarTestProfile3(Profile[VarTestStableStruct]): + B: List[uint16, 1024] + + +class ComplexTestProfile1(Profile[ComplexTestStableStruct]): + A: uint16 + B: List[uint16, 128] + C: uint8 + D: ByteList[256] + E: VarTestStableStruct + F: Vector[FixedTestStableStruct, 4] + G: Vector[VarTestStableStruct, 2] + + +class ComplexTestProfile2(Profile[ComplexTestStableStruct]): + A: uint16 + B: List[uint16, 128] + C: uint8 + D: ByteList[256] + E: VarTestStableStruct + + +class ComplexTestProfile3(Profile[ComplexTestStableStruct]): + A: uint16 + C: uint8 + E: VarTestStableStruct + G: Vector[VarTestStableStruct, 2] + + +class ComplexTestProfile4(Profile[ComplexTestStableStruct]): + B: List[uint16, 128] + D: ByteList[256] + F: Vector[FixedTestStableStruct, 4] + + +class ComplexTestProfile5(Profile[ComplexTestStableStruct]): + E: VarTestStableStruct + F: Vector[FixedTestStableStruct, 4] + G: Vector[VarTestStableStruct, 2] + + +class BitsProfile1(Profile[BitsStableStruct]): + A: Bitlist[5] + B: Bitvector[2] + C: Bitvector[1] + D: Bitlist[6] + E: Bitvector[8] + + +class BitsProfile2(Profile[BitsStableStruct]): + A: Bitlist[5] + B: Bitvector[2] + C: Bitvector[1] + D: Bitlist[6] + + +class BitsProfile3(Profile[BitsStableStruct]): + A: Bitlist[5] + D: Bitlist[6] + E: Bitvector[8] + + +def container_case_fn(rng: Random, mode: RandomizationMode, typ: Type[View], chaos: bool=False): + return get_random_ssz_object(rng, typ, + max_bytes_length=2000, + max_list_length=2000, + mode=mode, chaos=chaos) + + +PRESET_CONTAINERS: Dict[str, Tuple[Type[View], Sequence[int]]] = { + 'SingleFieldTestProfile': (SingleFieldTestProfile, []), + 'SmallTestProfile1': (SmallTestProfile1, []), + 'SmallTestProfile2': (SmallTestProfile2, []), + 'SmallTestProfile3': (SmallTestProfile3, []), + 'FixedTestProfile1': (FixedTestProfile1, []), + 'FixedTestProfile2': (FixedTestProfile2, []), + 'FixedTestProfile3': (FixedTestProfile3, []), + 'FixedTestProfile4': (FixedTestProfile4, []), + 'VarTestProfile1': (VarTestProfile1, [2]), + 'VarTestProfile2': (VarTestProfile2, [2]), + 'VarTestProfile3': (VarTestProfile3, [2]), + 'ComplexTestProfile1': (ComplexTestProfile1, [2, 2 + 4 + 1, 2 + 4 + 1 + 4]), + 'ComplexTestProfile2': (ComplexTestProfile2, [2, 2 + 4 + 1, 2 + 4 + 1 + 4]), + 'ComplexTestProfile3': (ComplexTestProfile3, [2, 2 + 4 + 1, 2 + 4 + 1 + 4]), + 'ComplexTestProfile4': (ComplexTestProfile4, [2, 2 + 4 + 1, 2 + 4 + 1 + 4]), + 'ComplexTestProfile5': (ComplexTestProfile5, [2, 2 + 4 + 1, 2 + 4 + 1 + 4]), + 'BitsProfile1': (BitsProfile1, [0, 4 + 1 + 1, 4 + 1 + 1 + 4]), + 'BitsProfile2': (BitsProfile2, [0, 4 + 1 + 1, 4 + 1 + 1 + 4]), + 'BitsProfile3': (BitsProfile3, [0, 4 + 1 + 1, 4 + 1 + 1 + 4]), +} + + +def valid_cases(): + rng = Random(1234) + for (name, (typ, offsets)) in PRESET_CONTAINERS.items(): + for mode in [RandomizationMode.mode_zero, RandomizationMode.mode_max]: + yield f'{name}_{mode.to_name()}', valid_test_case(lambda: container_case_fn(rng, mode, typ)) + + if len(offsets) == 0: + modes = [RandomizationMode.mode_random, RandomizationMode.mode_zero, RandomizationMode.mode_max] + else: + modes = list(RandomizationMode) + + for mode in modes: + for variation in range(3): + yield f'{name}_{mode.to_name()}_chaos_{variation}', \ + valid_test_case(lambda: container_case_fn(rng, mode, typ, chaos=True)) + # Notes: Below is the second wave of iteration, and only the random mode is selected + # for container without offset since ``RandomizationMode.mode_zero`` and ``RandomizationMode.mode_max`` + # are deterministic. + modes = [RandomizationMode.mode_random] if len(offsets) == 0 else list(RandomizationMode) + for mode in modes: + for variation in range(10): + yield f'{name}_{mode.to_name()}_{variation}', \ + valid_test_case(lambda: container_case_fn(rng, mode, typ)) + + +def mod_offset(b: bytes, offset_index: int, change: Callable[[int], int]): + return b[:offset_index] + \ + (change(int.from_bytes(b[offset_index:offset_index + 4], byteorder='little')) & 0xffffffff) \ + .to_bytes(length=4, byteorder='little') + \ + b[offset_index + 4:] + + +def invalid_cases(): + rng = Random(1234) + for (name, (typ, offsets)) in PRESET_CONTAINERS.items(): + # using mode_max_count, so that the extra byte cannot be picked up as normal list content + yield f'{name}_extra_byte', \ + invalid_test_case(lambda: serialize( + container_case_fn(rng, RandomizationMode.mode_max_count, typ)) + b'\xff') + + if len(offsets) != 0: + # Note: there are many more ways to have invalid offsets, + # these are just example to get clients started looking into hardening ssz. + for mode in [RandomizationMode.mode_random, + RandomizationMode.mode_nil_count, + RandomizationMode.mode_one_count, + RandomizationMode.mode_max_count]: + for index, offset_index in enumerate(offsets): + yield f'{name}_{mode.to_name()}_offset_{offset_index}_plus_one', \ + invalid_test_case(lambda: mod_offset( + b=serialize(container_case_fn(rng, mode, typ)), + offset_index=offset_index, + change=lambda x: x + 1 + )) + yield f'{name}_{mode.to_name()}_offset_{offset_index}_zeroed', \ + invalid_test_case(lambda: mod_offset( + b=serialize(container_case_fn(rng, mode, typ)), + offset_index=offset_index, + change=lambda x: 0 + )) + if index == 0: + yield f'{name}_{mode.to_name()}_offset_{offset_index}_minus_one', \ + invalid_test_case(lambda: mod_offset( + b=serialize(container_case_fn(rng, mode, typ)), + offset_index=offset_index, + change=lambda x: x - 1 + )) + if mode == RandomizationMode.mode_max_count: + serialized = serialize(container_case_fn(rng, mode, typ)) + serialized = serialized + serialized[:2] + yield f'{name}_{mode.to_name()}_last_offset_{offset_index}_overflow', \ + invalid_test_case(lambda: serialized) + if mode == RandomizationMode.mode_one_count: + serialized = serialize(container_case_fn(rng, mode, typ)) + serialized = serialized + serialized[:1] + yield f'{name}_{mode.to_name()}_last_offset_{offset_index}_wrong_byte_length', \ + invalid_test_case(lambda: serialized) diff --git a/tests/generators/ssz_generic/ssz_stablecontainer.py b/tests/generators/ssz_generic/ssz_stablecontainer.py new file mode 100644 index 0000000000..b4d782fee6 --- /dev/null +++ b/tests/generators/ssz_generic/ssz_stablecontainer.py @@ -0,0 +1,142 @@ +from ssz_test_case import invalid_test_case, valid_test_case +from eth2spec.utils.ssz.ssz_typing import View, byte, uint8, uint16, \ + uint32, uint64, List, ByteList, Vector, Bitvector, Bitlist, StableContainer +from eth2spec.utils.ssz.ssz_impl import serialize +from random import Random +from typing import Dict, Tuple, Sequence, Callable, Type, Optional +from eth2spec.debug.random_value import RandomizationMode, get_random_ssz_object + + +class SingleFieldTestStableStruct(StableContainer[4]): + A: Optional[byte] + + +class SmallTestStableStruct(StableContainer[4]): + A: Optional[uint16] + B: Optional[uint16] + + +class FixedTestStableStruct(StableContainer[4]): + A: Optional[uint8] + B: Optional[uint64] + C: Optional[uint32] + + +class VarTestStableStruct(StableContainer[4]): + A: Optional[uint16] + B: Optional[List[uint16, 1024]] + C: Optional[uint8] + + +class ComplexTestStableStruct(StableContainer[8]): + A: Optional[uint16] + B: Optional[List[uint16, 128]] + C: Optional[uint8] + D: Optional[ByteList[256]] + E: Optional[VarTestStableStruct] + F: Optional[Vector[FixedTestStableStruct, 4]] + G: Optional[Vector[VarTestStableStruct, 2]] + + +class BitsStableStruct(StableContainer[8]): + A: Optional[Bitlist[5]] + B: Optional[Bitvector[2]] + C: Optional[Bitvector[1]] + D: Optional[Bitlist[6]] + E: Optional[Bitvector[8]] + + +def container_case_fn(rng: Random, mode: RandomizationMode, typ: Type[View], chaos: bool=False): + return get_random_ssz_object(rng, typ, + max_bytes_length=2000, + max_list_length=2000, + mode=mode, chaos=chaos) + + +PRESET_CONTAINERS: Dict[str, Tuple[Type[View], Sequence[int]]] = { + 'SingleFieldTestStableStruct': (SingleFieldTestStableStruct, []), + 'SmallTestStableStruct': (SmallTestStableStruct, []), + 'FixedTestStableStruct': (FixedTestStableStruct, []), + 'VarTestStableStruct': (VarTestStableStruct, [2]), + 'ComplexTestStableStruct': (ComplexTestStableStruct, [2, 2 + 4 + 1, 2 + 4 + 1 + 4]), + 'BitsStableStruct': (BitsStableStruct, [0, 4 + 1 + 1, 4 + 1 + 1 + 4]), +} + + +def valid_cases(): + rng = Random(1234) + for (name, (typ, offsets)) in PRESET_CONTAINERS.items(): + for mode in [RandomizationMode.mode_zero, RandomizationMode.mode_max]: + yield f'{name}_{mode.to_name()}', valid_test_case(lambda: container_case_fn(rng, mode, typ)) + + if len(offsets) == 0: + modes = [RandomizationMode.mode_random, RandomizationMode.mode_zero, RandomizationMode.mode_max] + else: + modes = list(RandomizationMode) + + for mode in modes: + for variation in range(3): + yield f'{name}_{mode.to_name()}_chaos_{variation}', \ + valid_test_case(lambda: container_case_fn(rng, mode, typ, chaos=True)) + # Notes: Below is the second wave of iteration, and only the random mode is selected + # for container without offset since ``RandomizationMode.mode_zero`` and ``RandomizationMode.mode_max`` + # are deterministic. + modes = [RandomizationMode.mode_random] if len(offsets) == 0 else list(RandomizationMode) + for mode in modes: + for variation in range(10): + yield f'{name}_{mode.to_name()}_{variation}', \ + valid_test_case(lambda: container_case_fn(rng, mode, typ)) + + +def mod_offset(b: bytes, offset_index: int, change: Callable[[int], int]): + return b[:offset_index] + \ + (change(int.from_bytes(b[offset_index:offset_index + 4], byteorder='little')) & 0xffffffff) \ + .to_bytes(length=4, byteorder='little') + \ + b[offset_index + 4:] + + +def invalid_cases(): + rng = Random(1234) + for (name, (typ, offsets)) in PRESET_CONTAINERS.items(): + # using mode_max_count, so that the extra byte cannot be picked up as normal list content + yield f'{name}_extra_byte', \ + invalid_test_case(lambda: serialize( + container_case_fn(rng, RandomizationMode.mode_max_count, typ)) + b'\xff') + + if len(offsets) != 0: + # Note: there are many more ways to have invalid offsets, + # these are just example to get clients started looking into hardening ssz. + for mode in [RandomizationMode.mode_random, + RandomizationMode.mode_nil_count, + RandomizationMode.mode_one_count, + RandomizationMode.mode_max_count]: + for index, offset_index in enumerate(offsets): + yield f'{name}_{mode.to_name()}_offset_{offset_index}_plus_one', \ + invalid_test_case(lambda: mod_offset( + b=serialize(container_case_fn(rng, mode, typ)), + offset_index=offset_index, + change=lambda x: x + 1 + )) + yield f'{name}_{mode.to_name()}_offset_{offset_index}_zeroed', \ + invalid_test_case(lambda: mod_offset( + b=serialize(container_case_fn(rng, mode, typ)), + offset_index=offset_index, + change=lambda x: 0 + )) + if index == 0: + yield f'{name}_{mode.to_name()}_offset_{offset_index}_minus_one', \ + invalid_test_case(lambda: mod_offset( + b=serialize(container_case_fn(rng, mode, typ)), + offset_index=offset_index, + change=lambda x: x - 1 + )) + if mode == RandomizationMode.mode_max_count: + serialized = serialize(container_case_fn(rng, mode, typ)) + serialized = serialized + serialized[:2] + yield f'{name}_{mode.to_name()}_last_offset_{offset_index}_overflow', \ + invalid_test_case(lambda: serialized) + if mode == RandomizationMode.mode_one_count: + serialized = serialize(container_case_fn(rng, mode, typ)) + serialized = serialized + serialized[:1] + yield f'{name}_{mode.to_name()}_last_offset_{offset_index}_wrong_byte_length', \ + invalid_test_case(lambda: serialized)