diff --git a/.circleci/config.yml b/.circleci/config.yml index df415f2ebd..2767bec704 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -130,8 +130,11 @@ jobs: key: v3-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_pyspec_cached_venv - run: - name: Run linter + name: Run linter for pyspec command: make lint + - run: + name: Run linter for test generators + command: make lint_generators build_deposit_contract: docker: - image: ethereum/solc:0.6.11-alpine diff --git a/Makefile b/Makefile index 3914b7b275..8fa1044445 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,7 @@ SPEC_DIR = ./specs SSZ_DIR = ./ssz TEST_LIBS_DIR = ./tests/core +TEST_GENERATORS_DIR = ./tests/generators PY_SPEC_DIR = $(TEST_LIBS_DIR)/pyspec TEST_VECTOR_DIR = ../eth2.0-spec-tests/tests GENERATOR_DIR = ./tests/generators @@ -113,6 +114,10 @@ lint: pyspec flake8 --config $(LINTER_CONFIG_FILE) ./eth2spec \ && mypy --config-file $(LINTER_CONFIG_FILE) -p eth2spec.phase0 -p eth2spec.phase1 +lint_generators: pyspec + . venv/bin/activate; cd $(TEST_GENERATORS_DIR); \ + flake8 --config $(LINTER_CONFIG_FILE) + compile_deposit_contract: @cd $(SOLIDITY_DEPOSIT_CONTRACT_DIR) @git submodule update --recursive --init diff --git a/configs/mainnet/phase0.yaml b/configs/mainnet/phase0.yaml index d7d2f9b646..fbb12ba2ee 100644 --- a/configs/mainnet/phase0.yaml +++ b/configs/mainnet/phase0.yaml @@ -2,6 +2,7 @@ # Note: the intention of this file (for now) is to illustrate what a mainnet configuration could look like. # Some of these constants may still change before the launch of Phase 0. +CONFIG_NAME: "mainnet" # Misc # --------------------------------------------------------------- diff --git a/configs/mainnet/phase1.yaml b/configs/mainnet/phase1.yaml index 0e986079e2..1d689f2b56 100644 --- a/configs/mainnet/phase1.yaml +++ b/configs/mainnet/phase1.yaml @@ -1,5 +1,6 @@ # Mainnet preset - phase 1 +CONFIG_NAME: "mainnet" # phase1-fork # --------------------------------------------------------------- diff --git a/configs/minimal/phase0.yaml b/configs/minimal/phase0.yaml index 7008e1bf0c..0ee81738ef 100644 --- a/configs/minimal/phase0.yaml +++ b/configs/minimal/phase0.yaml @@ -1,5 +1,6 @@ # Minimal preset +CONFIG_NAME: "minimal" # Misc # --------------------------------------------------------------- diff --git a/configs/minimal/phase1.yaml b/configs/minimal/phase1.yaml index b9fce9dda9..cf17ba52a7 100644 --- a/configs/minimal/phase1.yaml +++ b/configs/minimal/phase1.yaml @@ -1,5 +1,6 @@ # Minimal preset - phase 1 +CONFIG_NAME: "minimal" # phase1-fork # --------------------------------------------------------------- diff --git a/setup.py b/setup.py index a051753007..b024e8d7f5 100644 --- a/setup.py +++ b/setup.py @@ -119,6 +119,8 @@ def get_spec(file_name: str) -> SpecObject: from eth2spec.utils.hash_function import hash SSZObject = TypeVar('SSZObject', bound=View) + +CONFIG_NAME = 'mainnet' ''' PHASE1_IMPORTS = '''from eth2spec.phase0 import spec as phase0 from eth2spec.config.config_util import apply_constants_config @@ -151,6 +153,8 @@ def get_spec(file_name: str) -> SpecObject: SSZVariableName = str GeneralizedIndex = NewType('GeneralizedIndex', int) SSZObject = TypeVar('SSZObject', bound=View) + +CONFIG_NAME = 'mainnet' ''' SUNDRY_CONSTANTS_FUNCTIONS = ''' def ceillog2(x: int) -> uint64: diff --git a/tests/core/gen_helpers/gen_base/gen_runner.py b/tests/core/gen_helpers/gen_base/gen_runner.py index 32f3594b35..a22073c00a 100644 --- a/tests/core/gen_helpers/gen_base/gen_runner.py +++ b/tests/core/gen_helpers/gen_base/gen_runner.py @@ -2,6 +2,7 @@ from pathlib import Path import sys from typing import Iterable, AnyStr, Any, Callable +import traceback from ruamel.yaml import ( YAML, @@ -9,6 +10,13 @@ from gen_base.gen_typing import TestProvider +from eth2spec.test import context +from eth2spec.test.exceptions import SkippedTest + + +# Flag that the runner does NOT run test via pytest +context.is_pytest = False + def validate_output_dir(path_str): path = Path(path_str) @@ -134,14 +142,20 @@ def output_part(out_kind: str, name: str, fn: Callable[[Path, ], None]): written_part = False meta = dict() - for (name, out_kind, data) in test_case.case_fn(): - written_part = True - if out_kind == "meta": - meta[name] = data - if out_kind == "data": - output_part("data", name, dump_yaml_fn(data, name, file_mode, yaml)) - if out_kind == "ssz": - output_part("ssz", name, dump_ssz_fn(data, name, file_mode)) + + try: + for (name, out_kind, data) in test_case.case_fn(): + written_part = True + if out_kind == "meta": + meta[name] = data + if out_kind == "data": + output_part("data", name, dump_yaml_fn(data, name, file_mode, yaml)) + if out_kind == "ssz": + output_part("ssz", name, dump_ssz_fn(data, name, file_mode)) + except SkippedTest as e: + print(e) + continue + # Once all meta data is collected (if any), write it to a meta data file. if len(meta) != 0: written_part = True @@ -152,6 +166,7 @@ def output_part(out_kind: str, name: str, fn: Callable[[Path, ], None]): except Exception as e: print(f"ERROR: failed to generate vector(s) for test {case_dir}: {e}") + traceback.print_exc() print(f"completed {generator_name}") diff --git a/tests/core/gen_helpers/requirements.txt b/tests/core/gen_helpers/requirements.txt index dc3f619042..e7cdd30ea2 100644 --- a/tests/core/gen_helpers/requirements.txt +++ b/tests/core/gen_helpers/requirements.txt @@ -1,2 +1,3 @@ ruamel.yaml==0.16.5 eth-utils==1.6.0 +pytest>=4.4 diff --git a/tests/core/gen_helpers/setup.py b/tests/core/gen_helpers/setup.py index b674dbfb6b..e9fc1a7877 100644 --- a/tests/core/gen_helpers/setup.py +++ b/tests/core/gen_helpers/setup.py @@ -5,6 +5,7 @@ packages=['gen_base', 'gen_from_tests'], install_requires=[ "ruamel.yaml==0.16.5", - "eth-utils==1.6.0" + "eth-utils==1.6.0", + "pytest>=4.4", ] ) diff --git a/tests/core/pyspec/eth2spec/config/config_util.py b/tests/core/pyspec/eth2spec/config/config_util.py index 1977e5b054..3f6764e98d 100644 --- a/tests/core/pyspec/eth2spec/config/config_util.py +++ b/tests/core/pyspec/eth2spec/config/config_util.py @@ -54,6 +54,8 @@ def load_config_file(configs_dir: str, presets_name: str) -> Dict[str, Any]: out[k] = [int(item) if item.isdigit() else item for item in v] elif isinstance(v, str) and v.startswith("0x"): out[k] = bytes.fromhex(v[2:]) + elif k == "CONFIG_NAME": + out[k] = str(v) else: out[k] = int(v) return out diff --git a/tests/core/pyspec/eth2spec/debug/random_value.py b/tests/core/pyspec/eth2spec/debug/random_value.py index 9fc3be9788..33eb25c0c6 100644 --- a/tests/core/pyspec/eth2spec/debug/random_value.py +++ b/tests/core/pyspec/eth2spec/debug/random_value.py @@ -68,9 +68,8 @@ def get_random_ssz_object(rng: Random, else: return typ(get_random_bytes_list(rng, rng.randint(0, min(max_bytes_length, typ.limit())))) if issubclass(typ, ByteVector): - # Sanity, don't generate absurdly big random values - # If a client is aiming to performance-test, they should create a benchmark suite. - assert typ.type_byte_length() <= max_bytes_length + # Random byte vectors can be bigger than max bytes size, e.g. custody chunk data. + # No max-bytes-length limitation here. if mode == RandomizationMode.mode_zero: return typ(b'\x00' * typ.type_byte_length()) elif mode == RandomizationMode.mode_max: diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 69be2fd0a5..cfd6724ed7 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -1,9 +1,11 @@ +import pytest + from eth2spec.phase0 import spec as spec_phase0 from eth2spec.phase1 import spec as spec_phase1 from eth2spec.utils import bls +from .exceptions import SkippedTest from .helpers.genesis import create_genesis_state - from .utils import vector_test, with_meta_tags from random import Random @@ -22,11 +24,16 @@ def reload_specs(): # Some of the Spec module functionality is exposed here to deal with phase-specific changes. SpecForkName = NewType("SpecForkName", str) +ConfigName = NewType("ConfigName", str) PHASE0 = SpecForkName('phase0') PHASE1 = SpecForkName('phase1') ALL_PHASES = (PHASE0, PHASE1) +MAINNET = ConfigName('mainnet') +MINIMAL = ConfigName('minimal') + + # TODO: currently phases are defined as python modules. # It would be better if they would be more well-defined interfaces for stronger typing. @@ -153,7 +160,7 @@ def low_single_balance(spec): def large_validator_set(spec): """ - Helper method to create a series of default balances. + Helper method to create a large series of default balances. Usage: `@with_custom_state(balances_fn=default_balances, ...)` """ num_validators = 2 * spec.SLOTS_PER_EPOCH * spec.MAX_COMMITTEES_PER_SLOT * spec.TARGET_COMMITTEE_SIZE @@ -184,6 +191,17 @@ def entry(*args, **kw): DEFAULT_BLS_ACTIVE = True +is_pytest = True + + +def dump_skipping_message(reason: str) -> None: + message = f"[Skipped test] {reason}" + if is_pytest: + pytest.skip(message) + else: + raise SkippedTest(message) + + def spec_test(fn): # Bls switch must be wrapped by vector_test, # to fully go through the yielded bls switch data, before setting back the BLS setting. @@ -255,6 +273,24 @@ def entry(*args, **kw): return entry +def disable_process_reveal_deadlines(fn): + """ + Decorator to make a function execute with `process_reveal_deadlines` OFF. + This is for testing long-range epochs transition without considering the reveal-deadline slashing effect. + """ + def entry(*args, spec: Spec, **kw): + if hasattr(spec, 'process_reveal_deadlines'): + old_state = spec.process_reveal_deadlines + spec.process_reveal_deadlines = lambda state: None + + yield from fn(*args, spec=spec, **kw) + + if hasattr(spec, 'process_reveal_deadlines'): + spec.process_reveal_deadlines = old_state + + return with_meta_tags({'reveal_deadlines_setting': 1})(entry) + + def with_all_phases(fn): """ A decorator for running a test with every phase @@ -284,7 +320,8 @@ def wrapper(*args, **kw): if 'phase' in kw: phase = kw.pop('phase') if phase not in phases: - return + dump_skipping_message(f"doesn't support this fork: {phase}") + return None run_phases = [phase] available_phases = set(run_phases) @@ -309,3 +346,33 @@ def wrapper(*args, **kw): return ret return wrapper return decorator + + +def with_configs(configs, reason=None): + def decorator(fn): + def wrapper(*args, spec: Spec, **kw): + available_configs = set(configs) + if spec.CONFIG_NAME not in available_configs: + message = f"doesn't support this config: {spec.CONFIG_NAME}." + if reason is not None: + message = f"{message} Reason: {reason}" + dump_skipping_message(message) + return None + + return fn(*args, spec=spec, **kw) + return wrapper + return decorator + + +def only_full_crosslink(fn): + def is_full_crosslink(spec, state): + epoch = spec.compute_epoch_at_slot(state.slot) + return spec.get_committee_count_per_slot(state, epoch) >= spec.get_active_shard_count(state) + + def wrapper(*args, spec: Spec, state: Any, **kw): + # TODO: update condition to "phase1+" if we have phase2 + if spec.fork == PHASE1 and not is_full_crosslink(spec, state): + dump_skipping_message("only for full crosslink") + return None + return fn(*args, spec=spec, state=state, **kw) + return wrapper diff --git a/tests/core/pyspec/eth2spec/test/exceptions.py b/tests/core/pyspec/eth2spec/test/exceptions.py new file mode 100644 index 0000000000..c553ec3744 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/exceptions.py @@ -0,0 +1,2 @@ +class SkippedTest(Exception): + ... diff --git a/tests/core/pyspec/eth2spec/test/helpers/custody.py b/tests/core/pyspec/eth2spec/test/helpers/custody.py index 162ca32639..b3a8c0a957 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/custody.py +++ b/tests/core/pyspec/eth2spec/test/helpers/custody.py @@ -179,15 +179,6 @@ def get_sample_shard_transition(spec, start_slot, block_lengths): return shard_transition -def get_custody_secret(spec, state, validator_index, epoch=None): - period = spec.get_custody_period_for_validator(validator_index, epoch if epoch is not None - else spec.get_current_epoch(state)) - epoch_to_sign = spec.get_randao_epoch_for_custody_period(period, validator_index) - domain = spec.get_domain(state, spec.DOMAIN_RANDAO, epoch_to_sign) - signing_root = spec.compute_signing_root(spec.Epoch(epoch_to_sign), domain) - return bls.Sign(privkeys[validator_index], signing_root) - - def get_custody_slashable_test_vector(spec, custody_secret, length, slashable=True): test_vector = get_custody_test_vector(length) offset = 0 diff --git a/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py b/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py index 6e508e3970..d10d1ee7bc 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py +++ b/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py @@ -35,8 +35,3 @@ def get_shard_transition_of_committee(spec, state, committee_index, shard_blocks shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) shard_transition = spec.get_shard_transition(state, shard, shard_blocks=shard_blocks) return shard_transition - - -def is_full_crosslink(spec, state): - epoch = spec.compute_epoch_at_slot(state.slot) - return spec.get_committee_count_per_slot(state, epoch) >= spec.get_active_shard_count(state) diff --git a/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_attester_slashing.py b/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_attester_slashing.py index 5f26ba5f78..82d4903117 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_attester_slashing.py +++ b/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_attester_slashing.py @@ -193,9 +193,6 @@ def test_participants_already_slashed(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing, False) -# Some of the following tests are phase0 only: phase 1 lists participants with bitfields instead of index list. - - @with_all_phases @spec_state_test @always_bls diff --git a/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py index 5b98abac48..721d68add1 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py @@ -21,10 +21,13 @@ from eth2spec.test.helpers.shard_transitions import get_shard_transition_of_committee from eth2spec.test.context import ( - PHASE0, PHASE1, - spec_test, - spec_state_test, with_all_phases, expect_assertion_error, always_bls, with_phases, - with_custom_state, single_phase, + PHASE0, PHASE1, MINIMAL, + spec_test, spec_state_test, dump_skipping_message, + with_phases, with_all_phases, single_phase, + expect_assertion_error, always_bls, + disable_process_reveal_deadlines, + with_configs, + with_custom_state, large_validator_set, ) @@ -90,6 +93,8 @@ def test_empty_block_transition(spec, state): @with_all_phases +@with_configs([MINIMAL], + reason="mainnet config leads to larger validator set than limit of public/private keys pre-generated") @spec_test @with_custom_state(balances_fn=large_validator_set, threshold_fn=lambda spec: spec.EJECTION_BALANCE) @single_phase @@ -316,6 +321,8 @@ def test_empty_epoch_transition(spec, state): @with_all_phases +@with_configs([MINIMAL], + reason="mainnet config leads to larger validator set than limit of public/private keys pre-generated") @spec_test @with_custom_state(balances_fn=large_validator_set, threshold_fn=lambda spec: spec.EJECTION_BALANCE) @single_phase @@ -338,10 +345,9 @@ def test_empty_epoch_transition_large_validator_set(spec, state): @with_all_phases @spec_state_test def test_empty_epoch_transition_not_finalizing(spec, state): - # Don't run for non-minimal configs, it takes very long, and the effect - # of calling finalization/justification is just the same as with the minimal configuration. if spec.SLOTS_PER_EPOCH > 8: - return + return dump_skipping_message("Skip mainnet config for saving time." + " Minimal config suffice to cover the target-of-test.") # copy for later balance lookups. pre_balances = list(state.balances) @@ -527,9 +533,8 @@ def test_attester_slashing(spec, state): @with_all_phases @spec_state_test def test_duplicate_attester_slashing(spec, state): - # Skip test if config cannot handle multiple AttesterSlashings per block if spec.MAX_ATTESTER_SLASHINGS < 2: - return + return dump_skipping_message("Skip test if config cannot handle multiple AttesterSlashings per block") attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=True) attester_slashings = [attester_slashing, attester_slashing.copy()] @@ -556,9 +561,8 @@ def test_duplicate_attester_slashing(spec, state): @with_all_phases @spec_state_test def test_multiple_attester_slashings_no_overlap(spec, state): - # Skip test if config cannot handle multiple AttesterSlashings per block if spec.MAX_ATTESTER_SLASHINGS < 2: - return + return dump_skipping_message("Skip test if config cannot handle multiple AttesterSlashings per block") # copy for later balance lookups. pre_state = state.copy() @@ -597,9 +601,8 @@ def test_multiple_attester_slashings_no_overlap(spec, state): @with_all_phases @spec_state_test def test_multiple_attester_slashings_partial_overlap(spec, state): - # Skip test if config cannot handle multiple AttesterSlashings per block if spec.MAX_ATTESTER_SLASHINGS < 2: - return + return dump_skipping_message("Skip test if config cannot handle multiple AttesterSlashings per block") # copy for later balance lookups. pre_state = state.copy() @@ -809,8 +812,9 @@ def create_signed_exit(index): # exceeding the minimal-config randao mixes memory size. # Applies to all voluntary-exit sanity block tests. -@with_phases([PHASE0]) +@with_all_phases @spec_state_test +@disable_process_reveal_deadlines def test_voluntary_exit(spec, state): validator_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] @@ -858,8 +862,9 @@ def test_double_validator_exit_same_block(spec, state): yield 'post', None -@with_phases([PHASE0]) +@with_all_phases @spec_state_test +@disable_process_reveal_deadlines def test_multiple_different_validator_exits_same_block(spec, state): validator_indices = [ spec.get_active_validator_indices(state, spec.get_current_epoch(state))[i] @@ -935,9 +940,9 @@ def test_historical_batch(spec, state): @with_all_phases @spec_state_test def test_eth1_data_votes_consensus(spec, state): - # Don't run when it will take very, very long to simulate. Minimal configuration suffices. if spec.EPOCHS_PER_ETH1_VOTING_PERIOD > 2: - return + return dump_skipping_message("Skip test if config with longer `EPOCHS_PER_ETH1_VOTING_PERIOD` for saving time." + " Minimal config suffice to cover the target-of-test.") voting_period_slots = spec.EPOCHS_PER_ETH1_VOTING_PERIOD * spec.SLOTS_PER_EPOCH @@ -979,9 +984,9 @@ def test_eth1_data_votes_consensus(spec, state): @with_all_phases @spec_state_test def test_eth1_data_votes_no_consensus(spec, state): - # Don't run when it will take very, very long to simulate. Minimal configuration suffices. if spec.EPOCHS_PER_ETH1_VOTING_PERIOD > 2: - return + return dump_skipping_message("Skip test if config with longer `EPOCHS_PER_ETH1_VOTING_PERIOD` for saving time." + " Minimal config suffice to cover the target-of-test.") voting_period_slots = spec.EPOCHS_PER_ETH1_VOTING_PERIOD * spec.SLOTS_PER_EPOCH diff --git a/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_chunk_challenge.py b/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_chunk_challenge.py index 9e67c0b480..e916010b25 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_chunk_challenge.py +++ b/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_chunk_challenge.py @@ -9,9 +9,12 @@ from eth2spec.test.helpers.state import transition_to, transition_to_valid_shard_slot from eth2spec.test.context import ( PHASE0, - with_all_phases_except, - spec_state_test, + MINIMAL, expect_assertion_error, + disable_process_reveal_deadlines, + spec_state_test, + with_all_phases_except, + with_configs, ) from eth2spec.test.phase0.block_processing.test_process_attestation import run_attestation_processing @@ -67,6 +70,8 @@ def run_custody_chunk_response_processing(spec, state, custody_response, valid=T @with_all_phases_except([PHASE0]) @spec_state_test +@with_configs([MINIMAL], reason="too slow") +@disable_process_reveal_deadlines def test_challenge_appended(spec, state): transition_to_valid_shard_slot(spec, state) transition_to(spec, state, state.slot + 1) # Make len(offset_slots) == 1 @@ -89,6 +94,8 @@ def test_challenge_appended(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test +@disable_process_reveal_deadlines +@with_configs([MINIMAL], reason="too slow") def test_challenge_empty_element_replaced(spec, state): transition_to_valid_shard_slot(spec, state) transition_to(spec, state, state.slot + 1) # Make len(offset_slots) == 1 @@ -113,6 +120,8 @@ def test_challenge_empty_element_replaced(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test +@disable_process_reveal_deadlines +@with_configs([MINIMAL], reason="too slow") def test_duplicate_challenge(spec, state): transition_to_valid_shard_slot(spec, state) transition_to(spec, state, state.slot + 1) # Make len(offset_slots) == 1 @@ -137,6 +146,8 @@ def test_duplicate_challenge(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test +@disable_process_reveal_deadlines +@with_configs([MINIMAL], reason="too slow") def test_second_challenge(spec, state): transition_to_valid_shard_slot(spec, state) transition_to(spec, state, state.slot + 1) # Make len(offset_slots) == 1 @@ -163,6 +174,8 @@ def test_second_challenge(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test +@disable_process_reveal_deadlines +@with_configs([MINIMAL], reason="too slow") def test_multiple_epochs_custody(spec, state): transition_to_valid_shard_slot(spec, state) transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * 3) @@ -186,6 +199,8 @@ def test_multiple_epochs_custody(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test +@disable_process_reveal_deadlines +@with_configs([MINIMAL], reason="too slow") def test_many_epochs_custody(spec, state): transition_to_valid_shard_slot(spec, state) transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * 20) @@ -209,6 +224,8 @@ def test_many_epochs_custody(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test +@disable_process_reveal_deadlines +@with_configs([MINIMAL], reason="too slow") def test_off_chain_attestation(spec, state): transition_to_valid_shard_slot(spec, state) transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH) @@ -228,6 +245,8 @@ def test_off_chain_attestation(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test +@disable_process_reveal_deadlines +@with_configs([MINIMAL], reason="too slow") def test_custody_response(spec, state): transition_to_valid_shard_slot(spec, state) transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH) @@ -258,6 +277,8 @@ def test_custody_response(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test +@disable_process_reveal_deadlines +@with_configs([MINIMAL], reason="too slow") def test_custody_response_chunk_index_2(spec, state): transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH) @@ -287,6 +308,8 @@ def test_custody_response_chunk_index_2(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test +@disable_process_reveal_deadlines +@with_configs([MINIMAL], reason="too slow") def test_custody_response_multiple_epochs(spec, state): transition_to_valid_shard_slot(spec, state) transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * 3) @@ -317,6 +340,8 @@ def test_custody_response_multiple_epochs(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test +@disable_process_reveal_deadlines +@with_configs([MINIMAL], reason="too slow") def test_custody_response_many_epochs(spec, state): transition_to_valid_shard_slot(spec, state) transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * 20) diff --git a/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_custody_slashing.py b/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_custody_slashing.py index b08b654130..fc7efa5bc9 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_custody_slashing.py +++ b/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_custody_slashing.py @@ -1,18 +1,21 @@ from eth2spec.test.helpers.custody import ( get_valid_custody_slashing, - get_custody_secret, get_custody_slashable_shard_transition, ) from eth2spec.test.helpers.attestations import ( get_valid_on_time_attestation, ) +from eth2spec.test.helpers.keys import privkeys from eth2spec.utils.ssz.ssz_typing import ByteList from eth2spec.test.helpers.state import get_balance, transition_to from eth2spec.test.context import ( PHASE0, + MINIMAL, with_all_phases_except, spec_state_test, expect_assertion_error, + disable_process_reveal_deadlines, + with_configs, ) from eth2spec.test.phase0.block_processing.test_process_attestation import run_attestation_processing @@ -77,7 +80,12 @@ def run_standard_custody_slashing_test(spec, if block_lengths is None: block_lengths = [2**15 // 3] * len(offset_slots) - custody_secret = get_custody_secret(spec, state, validator_index) + custody_secret = spec.get_custody_secret( + state, + validator_index, + privkeys[validator_index], + spec.get_current_epoch(state), + ) shard_transition, slashable_test_vector = get_custody_slashable_shard_transition( spec, state.slot, @@ -106,30 +114,40 @@ def run_standard_custody_slashing_test(spec, @with_all_phases_except([PHASE0]) @spec_state_test +@disable_process_reveal_deadlines +@with_configs([MINIMAL], reason="too slow") def test_custody_slashing(spec, state): yield from run_standard_custody_slashing_test(spec, state) @with_all_phases_except([PHASE0]) @spec_state_test +@disable_process_reveal_deadlines +@with_configs([MINIMAL], reason="too slow") def test_incorrect_custody_slashing(spec, state): yield from run_standard_custody_slashing_test(spec, state, correct=False) @with_all_phases_except([PHASE0]) @spec_state_test +@disable_process_reveal_deadlines +@with_configs([MINIMAL], reason="too slow") def test_multiple_epochs_custody(spec, state): yield from run_standard_custody_slashing_test(spec, state, shard_lateness=spec.SLOTS_PER_EPOCH * 3) @with_all_phases_except([PHASE0]) @spec_state_test +@disable_process_reveal_deadlines +@with_configs([MINIMAL], reason="too slow") def test_many_epochs_custody(spec, state): yield from run_standard_custody_slashing_test(spec, state, shard_lateness=spec.SLOTS_PER_EPOCH * 5) @with_all_phases_except([PHASE0]) @spec_state_test +@disable_process_reveal_deadlines +@with_configs([MINIMAL], reason="too slow") def test_invalid_custody_slashing(spec, state): yield from run_standard_custody_slashing_test( spec, diff --git a/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_shard_transition.py b/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_shard_transition.py index 2bb569eac2..b0a51557af 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_shard_transition.py +++ b/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_shard_transition.py @@ -1,6 +1,7 @@ from eth2spec.test.context import ( PHASE0, with_all_phases_except, + only_full_crosslink, spec_state_test, ) from eth2spec.test.helpers.attestations import ( @@ -10,7 +11,6 @@ ) from eth2spec.test.helpers.shard_transitions import ( run_shard_transitions_processing, - is_full_crosslink, ) from eth2spec.test.helpers.shard_block import ( build_shard_block, @@ -92,31 +92,22 @@ def run_successful_crosslink_tests(spec, state, target_len_offset_slot): @with_all_phases_except([PHASE0]) @spec_state_test +@only_full_crosslink def test_basic_crosslinks(spec, state): - if not is_full_crosslink(spec, state): - # Skip this test - return - yield from run_successful_crosslink_tests(spec, state, target_len_offset_slot=1) @with_all_phases_except([PHASE0]) @spec_state_test +@only_full_crosslink def test_multiple_offset_slots(spec, state): - if not is_full_crosslink(spec, state): - # Skip this test - return - yield from run_successful_crosslink_tests(spec, state, target_len_offset_slot=2) @with_all_phases_except([PHASE0]) @spec_state_test +@only_full_crosslink def test_no_winning_root(spec, state): - if not is_full_crosslink(spec, state): - # Skip this test - return - state, shard, target_shard_slot = get_initial_env(spec, state, target_len_offset_slot=1) init_slot = state.slot @@ -163,11 +154,8 @@ def test_no_winning_root(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test +@only_full_crosslink def test_wrong_shard_transition_root(spec, state): - if not is_full_crosslink(spec, state): - # Skip this test - return - state, shard, target_shard_slot = get_initial_env(spec, state, target_len_offset_slot=1) init_slot = state.slot diff --git a/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_challenge_deadlines.py b/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_challenge_deadlines.py index 828ace91cd..e270ff615e 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_challenge_deadlines.py +++ b/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_challenge_deadlines.py @@ -8,8 +8,10 @@ from eth2spec.test.helpers.state import transition_to, transition_to_valid_shard_slot from eth2spec.test.context import ( PHASE0, - with_all_phases_except, + MINIMAL, spec_state_test, + with_all_phases_except, + with_configs, ) from eth2spec.test.phase0.block_processing.test_process_attestation import run_attestation_processing from eth2spec.test.phase0.epoch_processing.run_epoch_process_base import run_epoch_processing_with @@ -25,6 +27,7 @@ def run_process_challenge_deadlines(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test +@with_configs([MINIMAL], reason="too slow") def test_validator_slashed_after_chunk_challenge(spec, state): transition_to_valid_shard_slot(spec, state) transition_to(spec, state, state.slot + 1) # Make len(offset_slots) == 1 diff --git a/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_reveal_deadlines.py b/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_reveal_deadlines.py index da1a604469..5777e184aa 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_reveal_deadlines.py +++ b/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_reveal_deadlines.py @@ -4,7 +4,9 @@ from eth2spec.test.helpers.state import transition_to from eth2spec.test.context import ( PHASE0, + MINIMAL, with_all_phases_except, + with_configs, spec_state_test, ) from eth2spec.test.phase0.epoch_processing.run_epoch_process_base import run_epoch_processing_with @@ -17,6 +19,7 @@ def run_process_challenge_deadlines(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test +@with_configs([MINIMAL], reason="too slow") def test_validator_slashed_after_reveal_deadline(spec, state): assert state.validators[0].slashed == 0 transition_to(spec, state, spec.get_randao_epoch_for_custody_period(0, 0) * spec.SLOTS_PER_EPOCH) @@ -36,6 +39,7 @@ def test_validator_slashed_after_reveal_deadline(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test +@with_configs([MINIMAL], reason="too slow") def test_validator_not_slashed_after_reveal(spec, state): transition_to(spec, state, spec.EPOCHS_PER_CUSTODY_PERIOD * spec.SLOTS_PER_EPOCH) custody_key_reveal = get_valid_custody_key_reveal(spec, state) diff --git a/tests/core/pyspec/eth2spec/test/phase1/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase1/sanity/test_blocks.py index 5fe3394db0..922b604ad4 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase1/sanity/test_blocks.py @@ -1,14 +1,15 @@ from typing import Dict, Sequence from eth2spec.test.context import ( - PHASE0, + PHASE0, MINIMAL, with_all_phases_except, spec_state_test, + only_full_crosslink, + with_configs, ) from eth2spec.test.helpers.attestations import get_valid_on_time_attestation from eth2spec.test.helpers.block import build_empty_block from eth2spec.test.helpers.custody import ( - get_custody_secret, get_custody_slashable_test_vector, get_valid_chunk_challenge, get_valid_custody_chunk_response, @@ -16,13 +17,13 @@ get_valid_custody_slashing, get_valid_early_derived_secret_reveal, ) +from eth2spec.test.helpers.keys import privkeys from eth2spec.test.helpers.shard_block import ( build_shard_block, get_committee_index_of_shard, get_sample_shard_block_body, get_shard_transitions, ) -from eth2spec.test.helpers.shard_transitions import is_full_crosslink from eth2spec.test.helpers.state import state_transition_and_sign_block, transition_to_valid_shard_slot, transition_to @@ -99,12 +100,8 @@ def run_beacon_block_with_shard_blocks(spec, state, target_len_offset_slot, comm @with_all_phases_except([PHASE0]) @spec_state_test +@only_full_crosslink def test_process_beacon_block_with_normal_shard_transition(spec, state): - # NOTE: this test is only for full crosslink (minimal config), not for mainnet - if not is_full_crosslink(spec, state): - # skip - return - transition_to_valid_shard_slot(spec, state) target_len_offset_slot = 1 @@ -117,12 +114,8 @@ def test_process_beacon_block_with_normal_shard_transition(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test +@only_full_crosslink def test_process_beacon_block_with_empty_proposal_transition(spec, state): - # NOTE: this test is only for full crosslink (minimal config), not for mainnet - if not is_full_crosslink(spec, state): - # skip - return - transition_to_valid_shard_slot(spec, state) target_len_offset_slot = 1 @@ -140,12 +133,8 @@ def test_process_beacon_block_with_empty_proposal_transition(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test +@only_full_crosslink def test_with_shard_transition_with_custody_challenge_and_response(spec, state): - # NOTE: this test is only for full crosslink (minimal config), not for mainnet - if not is_full_crosslink(spec, state): - # skip - return - transition_to_valid_shard_slot(spec, state) # build shard block @@ -178,6 +167,7 @@ def test_with_shard_transition_with_custody_challenge_and_response(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test +@with_configs([MINIMAL]) def test_custody_key_reveal(spec, state): transition_to_valid_shard_slot(spec, state) transition_to(spec, state, state.slot + spec.EPOCHS_PER_CUSTODY_PERIOD * spec.SLOTS_PER_EPOCH) @@ -202,12 +192,8 @@ def test_early_derived_secret_reveal(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test +@only_full_crosslink def test_custody_slashing(spec, state): - # NOTE: this test is only for full crosslink (minimal config), not for mainnet - if not is_full_crosslink(spec, state): - # skip - return - transition_to_valid_shard_slot(spec, state) # Build shard block @@ -215,7 +201,12 @@ def test_custody_slashing(spec, state): committee_index = get_committee_index_of_shard(spec, state, state.slot, shard) # Create slashable shard block body validator_index = spec.get_beacon_committee(state, state.slot, committee_index)[0] - custody_secret = get_custody_secret(spec, state, validator_index) + custody_secret = spec.get_custody_secret( + state, + validator_index, + privkeys[validator_index], + spec.get_current_epoch(state), + ) slashable_body = get_custody_slashable_test_vector(spec, custody_secret, length=100, slashable=True) shard_block = build_shard_block(spec, state, shard, body=slashable_body, slot=state.slot, signed=True) shard_block_dict: Dict[spec.Shard, Sequence[spec.SignedShardBlock]] = {shard: [shard_block]} diff --git a/tests/core/pyspec/eth2spec/test/phase1/sanity/test_shard_blocks.py b/tests/core/pyspec/eth2spec/test/phase1/sanity/test_shard_blocks.py index 440d0dd8d8..ab66314e5b 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/sanity/test_shard_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase1/sanity/test_shard_blocks.py @@ -4,12 +4,12 @@ expect_assertion_error, spec_state_test, with_all_phases_except, + only_full_crosslink, ) from eth2spec.test.helpers.shard_block import ( build_shard_block, sign_shard_block, ) -from eth2spec.test.helpers.shard_transitions import is_full_crosslink from eth2spec.test.helpers.state import next_slot, transition_to_valid_shard_slot, transition_to @@ -46,11 +46,8 @@ def run_shard_blocks(spec, shard_state, signed_shard_block, beacon_parent_state, @with_all_phases_except([PHASE0]) @spec_state_test @always_bls +@only_full_crosslink def test_valid_shard_block(spec, state): - if not is_full_crosslink(spec, state): - # skip - return - beacon_state = state.copy() transition_to_valid_shard_slot(spec, beacon_state) @@ -68,11 +65,8 @@ def test_valid_shard_block(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test +@only_full_crosslink def test_invalid_shard_parent_root(spec, state): - if not is_full_crosslink(spec, state): - # skip - return - beacon_state = state.copy() transition_to_valid_shard_slot(spec, beacon_state) @@ -87,11 +81,8 @@ def test_invalid_shard_parent_root(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test +@only_full_crosslink def test_invalid_beacon_parent_root(spec, state): - if not is_full_crosslink(spec, state): - # skip - return - beacon_state = state.copy() transition_to_valid_shard_slot(spec, beacon_state) shard = 0 @@ -105,11 +96,8 @@ def test_invalid_beacon_parent_root(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test +@only_full_crosslink def test_invalid_slot(spec, state): - if not is_full_crosslink(spec, state): - # skip - return - beacon_state = state.copy() transition_to_valid_shard_slot(spec, beacon_state) shard = 0 @@ -124,11 +112,8 @@ def test_invalid_slot(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test +@only_full_crosslink def test_invalid_proposer_index(spec, state): - if not is_full_crosslink(spec, state): - # skip - return - beacon_state = state.copy() transition_to_valid_shard_slot(spec, beacon_state) shard = 0 @@ -148,12 +133,8 @@ def test_invalid_proposer_index(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test @always_bls +@only_full_crosslink def test_out_of_bound_offset(spec, state): - # TODO: Handle this edge case properly - if not is_full_crosslink(spec, state): - # skip - return - beacon_state = state.copy() transition_to_valid_shard_slot(spec, beacon_state) shard = 0 @@ -173,11 +154,8 @@ def test_out_of_bound_offset(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test @always_bls +@only_full_crosslink def test_invalid_offset(spec, state): - if not is_full_crosslink(spec, state): - # skip - return - beacon_state = state.copy() transition_to_valid_shard_slot(spec, beacon_state) # 4 is not in `SHARD_BLOCK_OFFSETS` @@ -195,11 +173,8 @@ def test_invalid_offset(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test @always_bls +@only_full_crosslink def test_empty_block_body(spec, state): - if not is_full_crosslink(spec, state): - # skip - return - beacon_state = state.copy() transition_to_valid_shard_slot(spec, beacon_state) shard = 0 @@ -217,11 +192,8 @@ def test_empty_block_body(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test @always_bls +@only_full_crosslink def test_invalid_signature(spec, state): - if not is_full_crosslink(spec, state): - # skip - return - beacon_state = state.copy() transition_to_valid_shard_slot(spec, beacon_state) shard = 0 @@ -239,11 +211,8 @@ def test_invalid_signature(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test @always_bls +@only_full_crosslink def test_max_offset(spec, state): - if not is_full_crosslink(spec, state): - # skip - return - beacon_state = state.copy() transition_to_valid_shard_slot(spec, beacon_state) shard = 0 @@ -259,11 +228,8 @@ def test_max_offset(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test @always_bls +@only_full_crosslink def test_pending_shard_parent_block(spec, state): - if not is_full_crosslink(spec, state): - # skip - return - # Block N beacon_state = state.copy() transition_to_valid_shard_slot(spec, beacon_state) diff --git a/tests/core/pyspec/eth2spec/test/phase1/unittests/fork_choice/test_on_shard_block.py b/tests/core/pyspec/eth2spec/test/phase1/unittests/fork_choice/test_on_shard_block.py index ddf66b0f63..30eaa8d80b 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/unittests/fork_choice/test_on_shard_block.py +++ b/tests/core/pyspec/eth2spec/test/phase1/unittests/fork_choice/test_on_shard_block.py @@ -1,6 +1,6 @@ from eth2spec.utils.ssz.ssz_impl import hash_tree_root -from eth2spec.test.context import PHASE0, spec_state_test, with_all_phases_except, never_bls +from eth2spec.test.context import PHASE0, spec_state_test, with_all_phases_except, never_bls, only_full_crosslink from eth2spec.test.helpers.attestations import get_valid_on_time_attestation from eth2spec.test.helpers.shard_block import ( build_shard_block, @@ -8,7 +8,6 @@ get_committee_index_of_shard, ) from eth2spec.test.helpers.fork_choice import add_block_to_store, get_anchor_root, get_genesis_forkchoice_store -from eth2spec.test.helpers.shard_transitions import is_full_crosslink from eth2spec.test.helpers.state import state_transition_and_sign_block from eth2spec.test.helpers.block import build_empty_block @@ -209,11 +208,8 @@ def create_simple_fork(spec, state, store, shard): @with_all_phases_except([PHASE0]) @spec_state_test +@only_full_crosslink def test_shard_simple_fork(spec, state): - if not is_full_crosslink(spec, state): - # skip - return - spec.PHASE_1_GENESIS_SLOT = 0 # NOTE: mock genesis slot here state = spec.upgrade_to_phase1(state) shard = spec.Shard(1) @@ -237,11 +233,8 @@ def test_shard_simple_fork(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test +@only_full_crosslink def test_shard_latest_messages_for_different_shards(spec, state): - if not is_full_crosslink(spec, state): - # skip - return - spec.PHASE_1_GENESIS_SLOT = 0 # NOTE: mock genesis slot here state = spec.upgrade_to_phase1(state) shard_0 = spec.Shard(0) diff --git a/tests/formats/README.md b/tests/formats/README.md index 63b9a53903..36a5ec21b2 100644 --- a/tests/formats/README.md +++ b/tests/formats/README.md @@ -165,6 +165,9 @@ bls_setting: int -- optional, can have 3 different values: but there is no change of outcome when running the test if BLS is ON or OFF. 1: known as "BLS required" - if the test validity is strictly dependent on BLS being ON 2: known as "BLS ignored" - if the test validity is strictly dependent on BLS being OFF +reveal_deadlines_setting: -- optional, can have 2 different values: + 0: default, `process_reveal_deadlines` is ON. + 1: `process_reveal_deadlines` is OFF. ``` diff --git a/tests/formats/sanity/blocks.md b/tests/formats/sanity/blocks.md index 991bc35d22..44b37ed5ee 100644 --- a/tests/formats/sanity/blocks.md +++ b/tests/formats/sanity/blocks.md @@ -7,9 +7,10 @@ Sanity tests to cover a series of one or more blocks being processed, aiming to ### `meta.yaml` ```yaml -description: string -- Optional. Description of test case, purely for debugging purposes. -bls_setting: int -- see general test-format spec. -blocks_count: int -- the number of blocks processed in this test. +description: string -- Optional. Description of test case, purely for debugging purposes. +bls_setting: int -- see general test-format spec. +reveal_deadlines_setting: int -- see general test-format spec. +blocks_count: int -- the number of blocks processed in this test. ``` diff --git a/tests/generators/epoch_processing/main.py b/tests/generators/epoch_processing/main.py index 418d6c7505..fe8e0ee921 100644 --- a/tests/generators/epoch_processing/main.py +++ b/tests/generators/epoch_processing/main.py @@ -1,24 +1,17 @@ from typing import Iterable -from eth2spec.phase0 import spec as spec_phase0 -from eth2spec.phase1 import spec as spec_phase1 -from eth2spec.test.phase0.epoch_processing import ( - test_process_final_updates, - test_process_justification_and_finalization, - test_process_registry_updates, - test_process_rewards_and_penalties, - test_process_slashings -) from gen_base import gen_runner, gen_typing from gen_from_tests.gen import generate_from_tests -from importlib import reload +from importlib import reload, import_module from eth2spec.config import config_util -from eth2spec.test.context import PHASE0 +from eth2spec.phase0 import spec as spec_phase0 +from eth2spec.phase1 import spec as spec_phase1 +from eth2spec.test.context import PHASE0, PHASE1 from eth2spec.utils import bls -def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typing.TestProvider: - +def create_provider(fork_name: str, handler_name: str, + tests_src_mod_name: str, config_name: str) -> gen_typing.TestProvider: def prepare_fn(configs_path: str) -> str: config_util.prepare_config(configs_path, config_name) reload(spec_phase0) @@ -27,27 +20,40 @@ def prepare_fn(configs_path: str) -> str: return config_name def cases_fn() -> Iterable[gen_typing.TestCase]: + tests_src = import_module(tests_src_mod_name) return generate_from_tests( runner_name='epoch_processing', handler_name=handler_name, src=tests_src, - fork_name=PHASE0, + fork_name=fork_name, ) return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) if __name__ == "__main__": - gen_runner.run_generator("epoch_processing", [ - create_provider('final_updates', test_process_final_updates, 'minimal'), - create_provider('final_updates', test_process_final_updates, 'mainnet'), - create_provider('justification_and_finalization', test_process_justification_and_finalization, 'minimal'), - create_provider('justification_and_finalization', test_process_justification_and_finalization, 'mainnet'), - create_provider('registry_updates', test_process_registry_updates, 'minimal'), - create_provider('registry_updates', test_process_registry_updates, 'mainnet'), - create_provider('rewards_and_penalties', test_process_rewards_and_penalties, 'minimal'), - # runs full epochs filled with data, with uncached ssz. Disabled for now. - # create_provider('rewards_and_penalties', test_process_rewards_and_penalties, 'mainnet'), - create_provider('slashings', test_process_slashings, 'minimal'), - create_provider('slashings', test_process_slashings, 'mainnet'), + phase_0_mods = {key: 'eth2spec.test.phase0.epoch_processing.test_process_' + key for key in [ + 'final_updates', + 'justification_and_finalization', + 'registry_updates', + 'rewards_and_penalties', + 'slashings', + ]} + phase_1_mods = {**{key: 'eth2spec.test.phase1.epoch_processing.test_process_' + key for key in [ + 'challenge_deadlines', + 'custody_final_updates', + 'reveal_deadlines', + ]}, **phase_0_mods} # also run the previous phase 0 tests (but against phase 1 spec) + + gen_runner.run_generator(f"epoch_processing", [ + create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods.items() + ]) + gen_runner.run_generator(f"epoch_processing", [ + create_provider(PHASE0, key, mod_name, 'mainnet') for key, mod_name in phase_0_mods.items() + ]) + gen_runner.run_generator(f"epoch_processing", [ + create_provider(PHASE1, key, mod_name, 'minimal') for key, mod_name in phase_1_mods.items() + ]) + gen_runner.run_generator(f"epoch_processing", [ + create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods.items() ]) diff --git a/tests/generators/finality/main.py b/tests/generators/finality/main.py index dca0ecb8dc..ef2d8293f4 100644 --- a/tests/generators/finality/main.py +++ b/tests/generators/finality/main.py @@ -4,7 +4,7 @@ from gen_base import gen_runner, gen_typing from gen_from_tests.gen import generate_from_tests -from eth2spec.test.context import PHASE0 +from eth2spec.test.context import PHASE0, PHASE1 from eth2spec.test.phase0.finality import test_finality from eth2spec.config import config_util from eth2spec.phase0 import spec as spec_phase0 @@ -12,7 +12,7 @@ from eth2spec.utils import bls -def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typing.TestProvider: +def create_provider(fork_name: str, handler_name: str, tests_src, config_name: str) -> gen_typing.TestProvider: def prepare_fn(configs_path: str) -> str: config_util.prepare_config(configs_path, config_name) @@ -26,14 +26,18 @@ def cases_fn() -> Iterable[gen_typing.TestCase]: runner_name='finality', handler_name=handler_name, src=tests_src, - fork_name=PHASE0, + fork_name=fork_name, ) return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) if __name__ == "__main__": + # No additional phase 1 specific rewards tests, yet. + key = 'finality' gen_runner.run_generator("finality", [ - create_provider('finality', test_finality, 'minimal'), - create_provider('finality', test_finality, 'mainnet'), + create_provider(PHASE0, 'finality', test_finality, 'minimal'), + create_provider(PHASE0, 'finality', test_finality, 'mainnet'), + create_provider(PHASE1, 'finality', test_finality, 'minimal'), + create_provider(PHASE1, 'finality', test_finality, 'mainnet'), ]) diff --git a/tests/generators/operations/main.py b/tests/generators/operations/main.py index be490c5b2e..1acf45e477 100644 --- a/tests/generators/operations/main.py +++ b/tests/generators/operations/main.py @@ -1,26 +1,17 @@ from typing import Iterable -from eth2spec.test.phase0.block_processing import ( - test_process_attestation, - test_process_attester_slashing, - test_process_block_header, - test_process_deposit, - test_process_proposer_slashing, - test_process_voluntary_exit, -) - from gen_base import gen_runner, gen_typing from gen_from_tests.gen import generate_from_tests -from importlib import reload +from importlib import reload, import_module from eth2spec.config import config_util from eth2spec.phase0 import spec as spec_phase0 from eth2spec.phase1 import spec as spec_phase1 -from eth2spec.test.context import PHASE0 +from eth2spec.test.context import PHASE0, PHASE1 from eth2spec.utils import bls -def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typing.TestProvider: - +def create_provider(fork_name: str, handler_name: str, + tests_src_mod_name: str, config_name: str) -> gen_typing.TestProvider: def prepare_fn(configs_path: str) -> str: config_util.prepare_config(configs_path, config_name) reload(spec_phase0) @@ -29,28 +20,44 @@ def prepare_fn(configs_path: str) -> str: return config_name def cases_fn() -> Iterable[gen_typing.TestCase]: + tests_src = import_module(tests_src_mod_name) return generate_from_tests( runner_name='operations', handler_name=handler_name, src=tests_src, - fork_name=PHASE0, + fork_name=fork_name, ) return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) if __name__ == "__main__": - gen_runner.run_generator("operations", [ - create_provider('attestation', test_process_attestation, 'minimal'), - create_provider('attestation', test_process_attestation, 'mainnet'), - create_provider('attester_slashing', test_process_attester_slashing, 'minimal'), - create_provider('attester_slashing', test_process_attester_slashing, 'mainnet'), - create_provider('block_header', test_process_block_header, 'minimal'), - create_provider('block_header', test_process_block_header, 'mainnet'), - create_provider('deposit', test_process_deposit, 'minimal'), - create_provider('deposit', test_process_deposit, 'mainnet'), - create_provider('proposer_slashing', test_process_proposer_slashing, 'minimal'), - create_provider('proposer_slashing', test_process_proposer_slashing, 'mainnet'), - create_provider('voluntary_exit', test_process_voluntary_exit, 'minimal'), - create_provider('voluntary_exit', test_process_voluntary_exit, 'mainnet'), + phase_0_mods = {key: 'eth2spec.test.phase0.block_processing.test_process_' + key for key in [ + 'attestation', + 'attester_slashing', + 'block_header', + 'deposit', + 'proposer_slashing', + 'voluntary_exit', + ]} + phase_1_mods = {**{key: 'eth2spec.test.phase1.block_processing.test_process_' + key for key in [ + 'attestation', + 'chunk_challenge', + 'custody_key_reveal', + 'custody_slashing', + 'early_derived_secret_reveal', + 'shard_transition', + ]}, **phase_0_mods} # also run the previous phase 0 tests (but against phase 1 spec) + + gen_runner.run_generator(f"operations", [ + create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods.items() + ]) + gen_runner.run_generator(f"operations", [ + create_provider(PHASE0, key, mod_name, 'mainnet') for key, mod_name in phase_0_mods.items() + ]) + gen_runner.run_generator(f"operations", [ + create_provider(PHASE1, key, mod_name, 'minimal') for key, mod_name in phase_1_mods.items() + ]) + gen_runner.run_generator(f"operations", [ + create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods.items() ]) diff --git a/tests/generators/rewards/main.py b/tests/generators/rewards/main.py index dd82ee165a..23d0633b08 100644 --- a/tests/generators/rewards/main.py +++ b/tests/generators/rewards/main.py @@ -1,22 +1,17 @@ from typing import Iterable -from eth2spec.phase0 import spec as spec_phase0 -from eth2spec.phase1 import spec as spec_phase1 -from eth2spec.test.phase0.rewards import ( - test_basic, - test_leak, - test_random, -) from gen_base import gen_runner, gen_typing from gen_from_tests.gen import generate_from_tests -from importlib import reload +from importlib import reload, import_module from eth2spec.config import config_util -from eth2spec.test.context import PHASE0 +from eth2spec.phase0 import spec as spec_phase0 +from eth2spec.phase1 import spec as spec_phase1 +from eth2spec.test.context import PHASE0, PHASE1 from eth2spec.utils import bls -def create_provider(tests_src, config_name: str) -> gen_typing.TestProvider: - +def create_provider(fork_name: str, handler_name: str, + tests_src_mod_name: str, config_name: str) -> gen_typing.TestProvider: def prepare_fn(configs_path: str) -> str: config_util.prepare_config(configs_path, config_name) reload(spec_phase0) @@ -25,22 +20,35 @@ def prepare_fn(configs_path: str) -> str: return config_name def cases_fn() -> Iterable[gen_typing.TestCase]: + tests_src = import_module(tests_src_mod_name) return generate_from_tests( runner_name='rewards', - handler_name='core', + handler_name=handler_name, src=tests_src, - fork_name=PHASE0, + fork_name=fork_name, ) return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) if __name__ == "__main__": - gen_runner.run_generator("rewards", [ - create_provider(test_basic, 'minimal'), - create_provider(test_basic, 'mainnet'), - create_provider(test_leak, 'minimal'), - create_provider(test_leak, 'mainnet'), - create_provider(test_random, 'minimal'), - create_provider(test_random, 'mainnet'), + phase_0_mods = {key: 'eth2spec.test.phase0.rewards.test_' + key for key in [ + 'basic', + 'leak', + 'random', + ]} + # No additional phase 1 specific rewards tests, yet. + phase_1_mods = phase_0_mods + + gen_runner.run_generator(f"rewards", [ + create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods.items() + ]) + gen_runner.run_generator(f"rewards", [ + create_provider(PHASE0, key, mod_name, 'mainnet') for key, mod_name in phase_0_mods.items() + ]) + gen_runner.run_generator(f"rewards", [ + create_provider(PHASE1, key, mod_name, 'minimal') for key, mod_name in phase_1_mods.items() + ]) + gen_runner.run_generator(f"rewards", [ + create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods.items() ]) diff --git a/tests/generators/sanity/main.py b/tests/generators/sanity/main.py index 2feaaf09dd..83166f0cf0 100644 --- a/tests/generators/sanity/main.py +++ b/tests/generators/sanity/main.py @@ -1,19 +1,17 @@ from typing import Iterable -from importlib import reload from gen_base import gen_runner, gen_typing from gen_from_tests.gen import generate_from_tests - -from eth2spec.test.context import PHASE0 -from eth2spec.test.phase0.sanity import test_blocks, test_slots +from importlib import reload, import_module from eth2spec.config import config_util from eth2spec.phase0 import spec as spec_phase0 from eth2spec.phase1 import spec as spec_phase1 +from eth2spec.test.context import PHASE0, PHASE1 from eth2spec.utils import bls -def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typing.TestProvider: - +def create_provider(fork_name: str, handler_name: str, + tests_src_mod_name: str, config_name: str) -> gen_typing.TestProvider: def prepare_fn(configs_path: str) -> str: config_util.prepare_config(configs_path, config_name) reload(spec_phase0) @@ -22,20 +20,36 @@ def prepare_fn(configs_path: str) -> str: return config_name def cases_fn() -> Iterable[gen_typing.TestCase]: + tests_src = import_module(tests_src_mod_name) return generate_from_tests( runner_name='sanity', handler_name=handler_name, src=tests_src, - fork_name=PHASE0, + fork_name=fork_name, ) return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) if __name__ == "__main__": - gen_runner.run_generator("sanity", [ - create_provider('blocks', test_blocks, 'minimal'), - create_provider('blocks', test_blocks, 'mainnet'), - create_provider('slots', test_slots, 'minimal'), - create_provider('slots', test_slots, 'mainnet'), + phase_0_mods = {key: 'eth2spec.test.phase0.sanity.test_' + key for key in [ + 'blocks', + 'slots', + ]} + phase_1_mods = {**{key: 'eth2spec.test.phase1.sanity.test_' + key for key in [ + 'blocks', # more phase 1 specific block tests + 'shard_blocks', + ]}, **phase_0_mods} # also run the previous phase 0 tests (but against phase 1 spec) + + gen_runner.run_generator(f"sanity", [ + create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods.items() + ]) + gen_runner.run_generator(f"sanity", [ + create_provider(PHASE0, key, mod_name, 'mainnet') for key, mod_name in phase_0_mods.items() + ]) + gen_runner.run_generator(f"sanity", [ + create_provider(PHASE1, key, mod_name, 'minimal') for key, mod_name in phase_1_mods.items() + ]) + gen_runner.run_generator(f"sanity", [ + create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods.items() ]) diff --git a/tests/generators/ssz_generic/ssz_boolean.py b/tests/generators/ssz_generic/ssz_boolean.py index 9ff36ba88d..ec22c01be9 100644 --- a/tests/generators/ssz_generic/ssz_boolean.py +++ b/tests/generators/ssz_generic/ssz_boolean.py @@ -12,4 +12,3 @@ def invalid_cases(): yield "byte_rev_nibble", invalid_test_case(lambda: b'\x10') yield "byte_0x80", invalid_test_case(lambda: b'\x80') yield "byte_full", invalid_test_case(lambda: b'\xff') - diff --git a/tests/generators/ssz_generic/ssz_container.py b/tests/generators/ssz_generic/ssz_container.py index cf7c338398..9cd155f76b 100644 --- a/tests/generators/ssz_generic/ssz_container.py +++ b/tests/generators/ssz_generic/ssz_container.py @@ -85,7 +85,7 @@ def valid_cases(): def mod_offset(b: bytes, offset_index: int, change: Callable[[int], int]): return b[:offset_index] + \ (change(int.from_bytes(b[offset_index:offset_index + 4], byteorder='little')) & 0xffffffff) \ - .to_bytes(length=4, byteorder='little') + \ + .to_bytes(length=4, byteorder='little') + \ b[offset_index + 4:] diff --git a/tests/generators/ssz_static/main.py b/tests/generators/ssz_static/main.py index b9cb51db08..a30908fe9e 100644 --- a/tests/generators/ssz_static/main.py +++ b/tests/generators/ssz_static/main.py @@ -7,8 +7,9 @@ from eth2spec.debug import random_value, encode from eth2spec.config import config_util -from eth2spec.phase0 import spec -from eth2spec.test.context import PHASE0 +from eth2spec.phase0 import spec as spec_phase0 +from eth2spec.phase1 import spec as spec_phase1 +from eth2spec.test.context import PHASE0, PHASE1 from eth2spec.utils.ssz.ssz_typing import Container from eth2spec.utils.ssz.ssz_impl import ( hash_tree_root, @@ -16,11 +17,12 @@ ) -MAX_BYTES_LENGTH = 100 +MAX_BYTES_LENGTH = 1000 MAX_LIST_LENGTH = 10 -def create_test_case(rng: Random, typ, mode: random_value.RandomizationMode, chaos: bool) -> Iterable[gen_typing.TestCasePart]: +def create_test_case(rng: Random, typ, + mode: random_value.RandomizationMode, chaos: bool) -> Iterable[gen_typing.TestCasePart]: value = random_value.get_random_ssz_object(rng, typ, MAX_BYTES_LENGTH, MAX_LIST_LENGTH, mode, chaos) yield "value", "data", encode.encode(value) yield "serialized", "ssz", serialize(value) @@ -30,14 +32,15 @@ def create_test_case(rng: Random, typ, mode: random_value.RandomizationMode, cha yield "roots", "data", roots_data -def get_spec_ssz_types(): +def get_spec_ssz_types(spec): return [ (name, value) for (name, value) in getmembers(spec, isclass) if issubclass(value, Container) and value != Container # only the subclasses, not the imported base class ] -def ssz_static_cases(seed: int, name, ssz_type, mode: random_value.RandomizationMode, chaos: bool, count: int): +def ssz_static_cases(fork_name: str, seed: int, name, ssz_type, + mode: random_value.RandomizationMode, chaos: bool, count: int): random_mode_name = mode.to_name() # Reproducible RNG @@ -45,7 +48,7 @@ def ssz_static_cases(seed: int, name, ssz_type, mode: random_value.Randomization for i in range(count): yield gen_typing.TestCase( - fork_name=PHASE0, + fork_name=fork_name, runner_name='ssz_static', handler_name=name, suite_name=f"ssz_{random_mode_name}{'_chaos' if chaos else ''}", @@ -54,19 +57,23 @@ def ssz_static_cases(seed: int, name, ssz_type, mode: random_value.Randomization ) -def create_provider(config_name: str, seed: int, mode: random_value.RandomizationMode, chaos: bool, +def create_provider(fork_name, config_name: str, seed: int, mode: random_value.RandomizationMode, chaos: bool, cases_if_random: int) -> gen_typing.TestProvider: def prepare_fn(configs_path: str) -> str: # Apply changes to presets, this affects some of the vector types. config_util.prepare_config(configs_path, config_name) - reload(spec) + reload(spec_phase0) + reload(spec_phase1) return config_name def cases_fn() -> Iterable[gen_typing.TestCase]: count = cases_if_random if chaos or mode.is_changing() else 1 + spec = spec_phase0 + if fork_name == PHASE1: + spec = spec_phase1 - for (i, (name, ssz_type)) in enumerate(get_spec_ssz_types()): - yield from ssz_static_cases(seed * 1000 + i, name, ssz_type, mode, chaos, count) + for (i, (name, ssz_type)) in enumerate(get_spec_ssz_types(spec)): + yield from ssz_static_cases(fork_name, seed * 1000 + i, name, ssz_type, mode, chaos, count) return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) @@ -80,10 +87,11 @@ def cases_fn() -> Iterable[gen_typing.TestCase]: seed += 1 settings.append((seed, "minimal", random_value.RandomizationMode.mode_random, True, 30)) seed += 1 - settings.append((seed, "mainnet", random_value.RandomizationMode.mode_random, False, 5)) - seed += 1 - - gen_runner.run_generator("ssz_static", [ - create_provider(config_name, seed, mode, chaos, cases_if_random) - for (seed, config_name, mode, chaos, cases_if_random) in settings - ]) + # settings.append((seed, "mainnet", random_value.RandomizationMode.mode_random, False, 5)) + # seed += 1 + + for fork in [PHASE0, PHASE1]: + gen_runner.run_generator("ssz_static", [ + create_provider(fork, config_name, seed, mode, chaos, cases_if_random) + for (seed, config_name, mode, chaos, cases_if_random) in settings + ])