diff --git a/hathor/cli/run_node.py b/hathor/cli/run_node.py index a5f82f31d..9eb55abb1 100644 --- a/hathor/cli/run_node.py +++ b/hathor/cli/run_node.py @@ -22,7 +22,6 @@ from hathor.conf import TESTNET_SETTINGS_FILEPATH, HathorSettings from hathor.exception import PreInitializationError -from hathor.feature_activation.feature_service import FeatureService logger = get_logger() # LOGGING_CAPTURE_STDOUT = True @@ -150,6 +149,7 @@ def prepare(self, args: Namespace, *, register_resources: bool = True) -> None: from hathor.conf import HathorSettings settings = HathorSettings() + from hathor.feature_activation.feature_service import FeatureService feature_service = FeatureService( feature_settings=settings.FEATURE_ACTIVATION, tx_storage=self.manager.tx_storage diff --git a/hathor/feature_activation/feature_service.py b/hathor/feature_activation/feature_service.py index 6566d56bb..e07cc7d2d 100644 --- a/hathor/feature_activation/feature_service.py +++ b/hathor/feature_activation/feature_service.py @@ -35,30 +35,38 @@ def is_feature_active(self, *, block: Block, feature: Feature) -> bool: return state == FeatureState.ACTIVE def get_state(self, *, block: Block, feature: Feature) -> FeatureState: - """Returns the state of a feature at a certain block.""" + """Returns the state of a feature at a certain block. Uses block metadata to cache states.""" # per definition, the genesis block is in the DEFINED state for all features if block.is_genesis: return FeatureState.DEFINED + if state := block.get_feature_state(feature=feature): + return state + # All blocks within the same evaluation interval have the same state, that is, the state is only defined for - # the block in each interval boundary. Therefore, we get the state of the previous boundary. + # the block in each interval boundary. Therefore, we get the state of the previous boundary block or calculate + # a new state if this block is a boundary block. height = block.get_height() offset_to_boundary = height % self._feature_settings.evaluation_interval offset_to_previous_boundary = offset_to_boundary or self._feature_settings.evaluation_interval previous_boundary_height = height - offset_to_previous_boundary previous_boundary_block = self._get_ancestor_at_height(block=block, height=previous_boundary_height) - previous_state = self.get_state(block=previous_boundary_block, feature=feature) + previous_boundary_state = self.get_state(block=previous_boundary_block, feature=feature) if offset_to_boundary != 0: - return previous_state + return previous_boundary_state - return self._calculate_new_state( + new_state = self._calculate_new_state( boundary_block=block, feature=feature, - previous_state=previous_state + previous_state=previous_boundary_state ) + block.update_feature_state(feature=feature, state=new_state) + + return new_state + def _calculate_new_state( self, *, diff --git a/hathor/simulator/simulator.py b/hathor/simulator/simulator.py index c286eec46..e46033555 100644 --- a/hathor/simulator/simulator.py +++ b/hathor/simulator/simulator.py @@ -31,6 +31,7 @@ from hathor.simulator.miner.geometric_miner import GeometricMiner from hathor.simulator.tx_generator import RandomTransactionGenerator from hathor.transaction.genesis import _get_genesis_transactions_unsafe +from hathor.transaction.storage import TransactionStorage from hathor.util import Random from hathor.wallet import HDWallet @@ -149,7 +150,8 @@ def create_peer( enable_sync_v2: bool = True, soft_voided_tx_ids: Optional[Set[bytes]] = None, full_verification: bool = True, - event_ws_factory: Optional[EventWebsocketFactory] = None + event_ws_factory: Optional[EventWebsocketFactory] = None, + tx_storage: Optional[TransactionStorage] = None ) -> HathorManager: artifacts = self.create_peer_artifacts( network=network, @@ -159,6 +161,7 @@ def create_peer( soft_voided_tx_ids=soft_voided_tx_ids, full_verification=full_verification, event_ws_factory=event_ws_factory, + tx_storage=tx_storage ) return artifacts.manager @@ -171,7 +174,8 @@ def create_peer_artifacts( enable_sync_v2: bool = True, soft_voided_tx_ids: Optional[Set[bytes]] = None, full_verification: bool = True, - event_ws_factory: Optional[EventWebsocketFactory] = None + event_ws_factory: Optional[EventWebsocketFactory] = None, + tx_storage: Optional[TransactionStorage] = None ) -> BuildArtifacts: assert self._started, 'Simulator is not started.' assert peer_id is not None # XXX: temporary, for checking that tests are using the peer_id @@ -191,6 +195,9 @@ def create_peer_artifacts( .set_soft_voided_tx_ids(soft_voided_tx_ids or set()) \ .use_memory() + if tx_storage: + builder.set_tx_storage(tx_storage) + if event_ws_factory: builder.enable_event_manager(event_ws_factory=event_ws_factory) diff --git a/hathor/transaction/block.py b/hathor/transaction/block.py index 0f10ef618..1f1334c7b 100644 --- a/hathor/transaction/block.py +++ b/hathor/transaction/block.py @@ -21,6 +21,8 @@ from hathor import daa from hathor.checkpoint import Checkpoint from hathor.conf import HathorSettings +from hathor.feature_activation.feature import Feature +from hathor.feature_activation.model.feature_state import FeatureState from hathor.profiler import get_cpu_profiler from hathor.transaction import BaseTransaction, TxOutput, TxVersion from hathor.transaction.exceptions import ( @@ -415,3 +417,20 @@ def _get_feature_activation_bitmask(cls) -> int: bitmask = (1 << settings.FEATURE_ACTIVATION.max_signal_bits) - 1 return bitmask + + def get_feature_state(self, *, feature: Feature) -> Optional[FeatureState]: + """Returns the state of a feature from metadata.""" + metadata = self.get_metadata() + feature_states = metadata.feature_states or {} + + return feature_states.get(feature) + + def update_feature_state(self, *, feature: Feature, state: FeatureState) -> None: + """Updates the state of a feature in metadata and persists it.""" + assert self.storage is not None + metadata = self.get_metadata() + feature_states = metadata.feature_states or {} + feature_states[feature] = state + metadata.feature_states = feature_states + + self.storage.save_transaction(self, only_metadata=True) diff --git a/hathor/transaction/transaction_metadata.py b/hathor/transaction/transaction_metadata.py index 61a68af46..29cc38416 100644 --- a/hathor/transaction/transaction_metadata.py +++ b/hathor/transaction/transaction_metadata.py @@ -15,6 +15,8 @@ from collections import defaultdict from typing import TYPE_CHECKING, Any, Dict, FrozenSet, List, Optional, Set +from hathor.feature_activation.feature import Feature +from hathor.feature_activation.model.feature_state import FeatureState from hathor.transaction.validation_state import ValidationState from hathor.util import practically_equal @@ -49,6 +51,10 @@ class TransactionMetadata: # the previous boundary block up to this block, including it. LSB is on the left. feature_activation_bit_counts: Optional[list[int]] + # A dict of features in the feature activation process and their respective state. Must only be used by Blocks, + # is None otherwise. + feature_states: Optional[dict[Feature, FeatureState]] = None + # It must be a weakref. _tx_ref: Optional['ReferenceType[BaseTransaction]'] @@ -181,9 +187,9 @@ def __eq__(self, other: Any) -> bool: """Override the default Equals behavior""" if not isinstance(other, TransactionMetadata): return False - for field in ['hash', 'conflict_with', 'voided_by', 'received_by', - 'children', 'accumulated_weight', 'twins', 'score', - 'first_block', 'validation', 'min_height', 'feature_activation_bit_counts']: + for field in ['hash', 'conflict_with', 'voided_by', 'received_by', 'children', + 'accumulated_weight', 'twins', 'score', 'first_block', 'validation', + 'min_height', 'feature_activation_bit_counts', 'feature_states']: if (getattr(self, field) or None) != (getattr(other, field) or None): return False @@ -219,6 +225,10 @@ def to_json(self) -> Dict[str, Any]: data['height'] = self.height data['min_height'] = self.min_height data['feature_activation_bit_counts'] = self.feature_activation_bit_counts + + if self.feature_states is not None: + data['feature_states'] = {feature.value: state.value for feature, state in self.feature_states.items()} + if self.first_block is not None: data['first_block'] = self.first_block.hex() else: @@ -270,6 +280,13 @@ def create_from_json(cls, data: Dict[str, Any]) -> 'TransactionMetadata': meta.min_height = data.get('min_height', 0) meta.feature_activation_bit_counts = data.get('feature_activation_bit_counts', []) + feature_states_raw = data.get('feature_states') + if feature_states_raw: + meta.feature_states = { + Feature(feature): FeatureState(feature_state) + for feature, feature_state in feature_states_raw.items() + } + first_block_raw = data.get('first_block', None) if first_block_raw: meta.first_block = bytes.fromhex(first_block_raw) diff --git a/tests/feature_activation/test_feature_service.py b/tests/feature_activation/test_feature_service.py index 976e35c6e..ec3d6a1cf 100644 --- a/tests/feature_activation/test_feature_service.py +++ b/tests/feature_activation/test_feature_service.py @@ -364,6 +364,36 @@ def test_get_state_from_active(block_mocks: list[Block], tx_storage: Transaction assert result == FeatureState.ACTIVE +@pytest.mark.parametrize('block_height', [12, 13, 14, 15]) +def test_caching_mechanism(block_mocks: list[Block], tx_storage: TransactionStorage, block_height: int) -> None: + feature_settings = FeatureSettings.construct( + evaluation_interval=4, + features={ + Feature.NOP_FEATURE_1: Criteria.construct( + bit=Mock(), + start_height=0, + timeout_height=4, + activate_on_timeout=True, + version=Mock() + ) + } + ) + service = FeatureService(feature_settings=feature_settings, tx_storage=tx_storage) + block = block_mocks[block_height] + calculate_new_state_mock = Mock(wraps=service._calculate_new_state) + + with patch.object(FeatureService, '_calculate_new_state', calculate_new_state_mock): + result1 = service.get_state(block=block, feature=Feature.NOP_FEATURE_1) + + assert result1 == FeatureState.ACTIVE + assert calculate_new_state_mock.call_count == 3 + + result2 = service.get_state(block=block, feature=Feature.NOP_FEATURE_1) + + assert result2 == FeatureState.ACTIVE + assert calculate_new_state_mock.call_count == 3 + + @pytest.mark.parametrize('block_height', [12, 13, 14, 15]) def test_is_feature_active(block_mocks: list[Block], tx_storage: TransactionStorage, block_height: int) -> None: feature_settings = FeatureSettings.construct( diff --git a/tests/feature_activation/test_feature_simulation.py b/tests/feature_activation/test_feature_simulation.py index 18838947b..d2a3c0aab 100644 --- a/tests/feature_activation/test_feature_simulation.py +++ b/tests/feature_activation/test_feature_simulation.py @@ -15,16 +15,22 @@ from typing import Any from unittest.mock import Mock, patch -from hathor.feature_activation import feature_service +import pytest + +from hathor.feature_activation import feature_service as feature_service_module from hathor.feature_activation.feature import Feature +from hathor.feature_activation.feature_service import FeatureService from hathor.feature_activation.model.criteria import Criteria from hathor.feature_activation.resources.feature import FeatureResource from hathor.feature_activation.settings import Settings as FeatureSettings from hathor.p2p.peer_id import PeerId +from hathor.simulator.miner import AbstractMiner from hathor.simulator.trigger import StopAfterNMinedBlocks +from hathor.storage import RocksDBStorage from tests import unittest from tests.resources.base_resource import StubSite from tests.simulation.base import SimulatorTestCase +from tests.utils import HAS_ROCKSDB _FEATURE_SETTINGS = FeatureSettings( evaluation_interval=4, @@ -42,47 +48,56 @@ ) -class BaseTestFeatureSimulation(SimulatorTestCase): +class BaseFeatureSimulationTest(SimulatorTestCase): seed_config = 5604003716498505253 - def setUp(self): - super().setUp() + def _get_result_after(self, *, n_blocks: int, miner: AbstractMiner, web_client: StubSite) -> dict[str, Any]: + """Returns the feature activation api response after N blocks.""" + trigger = StopAfterNMinedBlocks(miner, quantity=n_blocks) + self.simulator.run(36000, trigger=trigger) + + response = web_client.get('feature') + result = response.result.json_value() + + return result + + @staticmethod + def _get_state_mock_block_height_calls(get_state_mock: Mock) -> list[int]: + """Returns the heights of blocks that get_state_mock was called with.""" + return [call.kwargs['block'].get_height() for call in get_state_mock.call_args_list] + + def test_feature(self): + """ + Tests that a feature goes through all possible states in the correct block heights, and also assert internal + method call counts and args to make sure we're executing it in the most performatic way. + """ peer_id = PeerId() - artifacts = self.simulator.create_peer_artifacts(peer_id=peer_id) + artifacts = self.simulator.create_peer_artifacts(peer_id=peer_id, tx_storage=self.tx_storage) manager = artifacts.manager manager.allow_mining_without_peers() - self.feature_service = artifacts.feature_service - self.feature_service._feature_settings = _FEATURE_SETTINGS + feature_service = artifacts.feature_service + feature_service._feature_settings = _FEATURE_SETTINGS feature_resource = FeatureResource( feature_settings=_FEATURE_SETTINGS, - feature_service=self.feature_service, - tx_storage=manager.tx_storage + feature_service=feature_service, + tx_storage=self.tx_storage ) - self.web_client = StubSite(feature_resource) - - self.miner = self.simulator.create_miner(manager, hashpower=1e6) - self.miner.start() + web_client = StubSite(feature_resource) - def _get_result_after(self, *, n_blocks: int) -> dict[str, Any]: - trigger = StopAfterNMinedBlocks(self.miner, quantity=n_blocks) - self.simulator.run(3600, trigger=trigger) + miner = self.simulator.create_miner(manager, hashpower=1e6) + miner.start() - response = self.web_client.get('feature') - result = response.result.json_value() - - return result - - def test_feature(self): - """ - Test that a feature goes through all possible states in the correct block heights, and also assert internal - method call counts to make sure we're executing it in the most performatic way. - """ - get_ancestor_iteratively_mock = Mock(wraps=feature_service._get_ancestor_iteratively) + get_state_mock = Mock(wraps=feature_service.get_state) + get_ancestor_iteratively_mock = Mock(wraps=feature_service_module._get_ancestor_iteratively) - with patch.object(feature_service, '_get_ancestor_iteratively', get_ancestor_iteratively_mock): - # at the beginning, the feature is DEFINED - assert self._get_result_after(n_blocks=10) == dict( + with ( + patch.object(FeatureService, 'get_state', get_state_mock), + patch.object(feature_service_module, '_get_ancestor_iteratively', get_ancestor_iteratively_mock) + ): + # at the beginning, the feature is DEFINED: + result = self._get_result_after(n_blocks=10, miner=miner, web_client=web_client) + assert result == dict( block_hash='6a0552f08705978048bc8981be718a04aad61fb14f13f19155f1081996bc6321', block_height=10, features=[ @@ -99,11 +114,15 @@ def test_feature(self): ) ] ) - # no blocks are voided, so we only use the height index: + # so we query states all the way down to genesis: + assert self._get_state_mock_block_height_calls(get_state_mock) == [10, 8, 4, 0] + # no blocks are voided, so we only use the height index, and not get_ancestor_iteratively: assert get_ancestor_iteratively_mock.call_count == 0 + get_state_mock.reset_mock() - # at block 19, the feature is DEFINED, just before becoming STARTED - assert self._get_result_after(n_blocks=9) == dict( + # at block 19, the feature is DEFINED, just before becoming STARTED: + result = self._get_result_after(n_blocks=9, miner=miner, web_client=web_client) + assert result == dict( block_hash='faa3c7715f29f7b325c7830e26fb84d1a3059bf24ee3bfbdd021296fb205f265', block_height=19, features=[ @@ -120,10 +139,14 @@ def test_feature(self): ) ] ) + # so we query states from block 19 to 8, as it's cached: + assert self._get_state_mock_block_height_calls(get_state_mock) == [19, 16, 12, 8] assert get_ancestor_iteratively_mock.call_count == 0 + get_state_mock.reset_mock() - # at block 20, the feature becomes STARTED - assert self._get_result_after(n_blocks=1) == dict( + # at block 20, the feature becomes STARTED: + result = self._get_result_after(n_blocks=1, miner=miner, web_client=web_client) + assert result == dict( block_hash='f29544442676671d6b20f2c8cf2247a00b63b07ed3c3b853cd7f3f85f260c981', block_height=20, features=[ @@ -140,10 +163,13 @@ def test_feature(self): ) ] ) + assert self._get_state_mock_block_height_calls(get_state_mock) == [20, 16] assert get_ancestor_iteratively_mock.call_count == 0 + get_state_mock.reset_mock() - # at block 39, the feature is STARTED, just before becoming ACTIVE - assert self._get_result_after(n_blocks=39) == dict( + # at block 39, the feature is STARTED, just before becoming ACTIVE: + result = self._get_result_after(n_blocks=39, miner=miner, web_client=web_client) + assert result == dict( block_hash='bc5dc312e4263ab3e7b94a26042704e870d86aa00878204f9f996eab9e384387', block_height=59, features=[ @@ -160,10 +186,15 @@ def test_feature(self): ) ] ) + assert ( + self._get_state_mock_block_height_calls(get_state_mock) == [59, 56, 52, 48, 44, 40, 36, 32, 28, 24, 20] + ) assert get_ancestor_iteratively_mock.call_count == 0 + get_state_mock.reset_mock() - # at block 60, the feature becomes ACTIVE, forever - assert self._get_result_after(n_blocks=1) == dict( + # at block 60, the feature becomes ACTIVE, forever: + result = self._get_result_after(n_blocks=1, miner=miner, web_client=web_client) + assert result == dict( block_hash='df85ad722265b9b50394c6920cddbdaf0727057036cabce6414f591b82eed22b', block_height=60, features=[ @@ -180,17 +211,169 @@ def test_feature(self): ) ] ) + assert self._get_state_mock_block_height_calls(get_state_mock) == [60, 56] assert get_ancestor_iteratively_mock.call_count == 0 + get_state_mock.reset_mock() + + def test_feature_from_existing_storage(self): + """ + Tests that feature states are correctly retrieved from an existing storage, so no recalculation is required. + """ + peer_id = PeerId() + artifacts1 = self.simulator.create_peer_artifacts(peer_id=peer_id, tx_storage=self.tx_storage) + manager1 = artifacts1.manager + manager1.allow_mining_without_peers() + + feature_service = artifacts1.feature_service + feature_service._feature_settings = _FEATURE_SETTINGS + feature_resource = FeatureResource( + feature_settings=_FEATURE_SETTINGS, + feature_service=feature_service, + tx_storage=self.tx_storage + ) + web_client = StubSite(feature_resource) + + miner = self.simulator.create_miner(manager1, hashpower=1e6) + miner.start() + + get_state_mock = Mock(wraps=feature_service.get_state) + get_ancestor_iteratively_mock = Mock(wraps=feature_service_module._get_ancestor_iteratively) + + with ( + patch.object(FeatureService, 'get_state', get_state_mock), + patch.object(feature_service_module, '_get_ancestor_iteratively', get_ancestor_iteratively_mock) + ): + assert self.tx_storage.get_vertices_count() == 3 # genesis vertices in the storage + + result = self._get_result_after(n_blocks=60, miner=miner, web_client=web_client) + assert result == dict( + block_hash='df85ad722265b9b50394c6920cddbdaf0727057036cabce6414f591b82eed22b', + block_height=60, + features=[ + dict( + name='NOP_FEATURE_1', + state='ACTIVE', + acceptance=None, + threshold=0.75, + start_height=20, + timeout_height=60, + minimum_activation_height=0, + activate_on_timeout=True, + version='0.0.0' + ) + ] + ) + # feature states have to be calculated for all blocks in evaluation interval boundaries, as this is the + # first run: + assert self._get_state_mock_block_height_calls(get_state_mock) == list(range(60, -4, -4)) + # no blocks are voided, so we only use the height index: + assert get_ancestor_iteratively_mock.call_count == 0 + assert self.tx_storage.get_vertices_count() == 63 + get_state_mock.reset_mock() + + miner.stop() + manager1.stop() + + artifacts2 = self.simulator.create_peer_artifacts( + peer_id=peer_id, + tx_storage=self.tx_storage, + full_verification=False + ) + + # new feature_service is created with the same storage generated above + feature_service = artifacts2.feature_service + feature_service._feature_settings = _FEATURE_SETTINGS + feature_resource = FeatureResource( + feature_settings=_FEATURE_SETTINGS, + feature_service=feature_service, + tx_storage=self.tx_storage + ) + web_client = StubSite(feature_resource) + + get_state_mock = Mock(wraps=feature_service.get_state) + get_ancestor_iteratively_mock = Mock(wraps=feature_service_module._get_ancestor_iteratively) + + with ( + patch.object(FeatureService, 'get_state', get_state_mock), + patch.object(feature_service_module, '_get_ancestor_iteratively', get_ancestor_iteratively_mock) + ): + # the new storage starts populated + assert self.tx_storage.get_vertices_count() == 63 + self.simulator.run(3600) + + response = web_client.get('feature') + result = response.result.json_value() + + assert result == dict( + block_hash='df85ad722265b9b50394c6920cddbdaf0727057036cabce6414f591b82eed22b', + block_height=60, + features=[ + dict( + name='NOP_FEATURE_1', + state='ACTIVE', + acceptance=None, + threshold=0.75, + start_height=20, + timeout_height=60, + minimum_activation_height=0, + activate_on_timeout=True, + version='0.0.0' + ) + ] + ) + # features states are not queried for previous blocks, as they have it cached: + assert self._get_state_mock_block_height_calls(get_state_mock) == [60] + assert get_ancestor_iteratively_mock.call_count == 0 + assert self.tx_storage.get_vertices_count() == 63 + get_state_mock.reset_mock() + + +class BaseMemoryStorageFeatureSimulationTest(BaseFeatureSimulationTest): + def setUp(self): + super().setUp() + from hathor.transaction.storage import TransactionMemoryStorage + self.tx_storage = TransactionMemoryStorage() + + +@pytest.mark.skipif(not HAS_ROCKSDB, reason='requires python-rocksdb') +class BaseRocksDBStorageFeatureSimulationTest(BaseFeatureSimulationTest): + def setUp(self): + super().setUp() + import tempfile + + from hathor.transaction.storage import TransactionRocksDBStorage + + directory = tempfile.mkdtemp() + self.tmpdirs.append(directory) + + rocksdb_storage = RocksDBStorage(path=directory) + self.tx_storage = TransactionRocksDBStorage(rocksdb_storage) + + +class SyncV1MemoryStorageFeatureSimulationTest(unittest.SyncV1Params, BaseMemoryStorageFeatureSimulationTest): + __test__ = True + + +class SyncV2MemoryStorageFeatureSimulationTest(unittest.SyncV2Params, BaseMemoryStorageFeatureSimulationTest): + __test__ = True + + +# sync-bridge should behave like sync-v2 +class SyncBridgeMemoryStorageFeatureSimulationTest(unittest.SyncBridgeParams, BaseMemoryStorageFeatureSimulationTest): + __test__ = True -class SyncV1BaseTestFeatureSimulation(unittest.SyncV1Params, BaseTestFeatureSimulation): +class SyncV1RocksDBStorageFeatureSimulationTest(unittest.SyncV1Params, BaseRocksDBStorageFeatureSimulationTest): __test__ = True -class SyncV2BaseTestFeatureSimulation(unittest.SyncV2Params, BaseTestFeatureSimulation): +class SyncV2RocksDBStorageFeatureSimulationTest(unittest.SyncV2Params, BaseRocksDBStorageFeatureSimulationTest): __test__ = True # sync-bridge should behave like sync-v2 -class SyncBridgeBaseTestFeatureSimulation(unittest.SyncBridgeParams, SyncV2BaseTestFeatureSimulation): +class SyncBridgeRocksDBStorageFeatureSimulationTest( + unittest.SyncBridgeParams, + BaseRocksDBStorageFeatureSimulationTest +): __test__ = True