Merge pull request #995 from HathorNetwork/master

Release-candidate v0.60.0-rc.1

hathor/builder/builder.py

@@ -16,6 +16,7 @@
 from typing import Any, Callable, NamedTuple, Optional, TypeAlias
 
 from structlog import get_logger
+from typing_extensions import assert_never
 
 from hathor.checkpoint import Checkpoint
 from hathor.conf.get_settings import get_global_settings
@@ -25,9 +26,11 @@
 from hathor.event import EventManager
 from hathor.event.storage import EventMemoryStorage, EventRocksDBStorage, EventStorage
 from hathor.event.websocket import EventWebsocketFactory
+from hathor.execution_manager import ExecutionManager
 from hathor.feature_activation.bit_signaling_service import BitSignalingService
 from hathor.feature_activation.feature import Feature
 from hathor.feature_activation.feature_service import FeatureService
+from hathor.feature_activation.storage.feature_activation_storage import FeatureActivationStorage
 from hathor.indexes import IndexesManager, MemoryIndexesManager, RocksDBIndexesManager
 from hathor.manager import HathorManager
 from hathor.mining.cpu_mining_service import CpuMiningService
@@ -44,7 +47,8 @@
     TransactionStorage,
 )
 from hathor.util import Random, get_environment_info, not_none
-from hathor.verification.verification_service import VerificationService, VertexVerifiers
+from hathor.verification.verification_service import VerificationService
+from hathor.verification.vertex_verifiers import VertexVerifiers
 from hathor.wallet import BaseWallet, Wallet
 
 logger = get_logger()
@@ -67,6 +71,7 @@ class BuildArtifacts(NamedTuple):
     consensus: ConsensusAlgorithm
     tx_storage: TransactionStorage
     feature_service: FeatureService
+    bit_signaling_service: BitSignalingService
     indexes: Optional[IndexesManager]
     wallet: Optional[BaseWallet]
     rocksdb_storage: Optional[RocksDBStorage]
@@ -150,6 +155,8 @@ def __init__(self) -> None:
 
         self._soft_voided_tx_ids: Optional[set[bytes]] = None
 
+        self._execution_manager: ExecutionManager | None = None
+
     def build(self) -> BuildArtifacts:
         if self.artifacts is not None:
             raise ValueError('cannot call build twice')
@@ -163,8 +170,9 @@ def build(self) -> BuildArtifacts:
 
         peer_id = self._get_peer_id()
 
+        execution_manager = self._get_or_create_execution_manager()
         soft_voided_tx_ids = self._get_soft_voided_tx_ids()
-        consensus_algorithm = ConsensusAlgorithm(soft_voided_tx_ids, pubsub)
+        consensus_algorithm = ConsensusAlgorithm(soft_voided_tx_ids, pubsub, execution_manager=execution_manager)
 
         p2p_manager = self._get_p2p_manager()
 
@@ -215,6 +223,7 @@ def build(self) -> BuildArtifacts:
             bit_signaling_service=bit_signaling_service,
             verification_service=verification_service,
             cpu_mining_service=cpu_mining_service,
+            execution_manager=execution_manager,
             **kwargs
         )
 
@@ -239,6 +248,7 @@ def build(self) -> BuildArtifacts:
             rocksdb_storage=self._rocksdb_storage,
             stratum_factory=stratum_factory,
             feature_service=feature_service,
+            bit_signaling_service=bit_signaling_service
         )
 
         return self.artifacts
@@ -306,6 +316,13 @@ def _get_peer_id(self) -> PeerId:
             return self._peer_id
         raise ValueError('peer_id not set')
 
+    def _get_or_create_execution_manager(self) -> ExecutionManager:
+        if self._execution_manager is None:
+            reactor = self._get_reactor()
+            self._execution_manager = ExecutionManager(reactor)
+
+        return self._execution_manager
+
     def _get_or_create_pubsub(self) -> PubSubManager:
         if self._pubsub is None:
             self._pubsub = PubSubManager(self._get_reactor())
@@ -438,7 +455,8 @@ def _get_or_create_event_manager(self) -> EventManager:
                 reactor=reactor,
                 pubsub=self._get_or_create_pubsub(),
                 event_storage=storage,
-                event_ws_factory=factory
+                event_ws_factory=factory,
+                execution_manager=self._get_or_create_execution_manager()
             )
 
         return self._event_manager
@@ -460,12 +478,14 @@ def _get_or_create_bit_signaling_service(self) -> BitSignalingService:
             settings = self._get_or_create_settings()
             tx_storage = self._get_or_create_tx_storage()
             feature_service = self._get_or_create_feature_service()
+            feature_storage = self._get_or_create_feature_storage()
             self._bit_signaling_service = BitSignalingService(
                 feature_settings=settings.FEATURE_ACTIVATION,
                 feature_service=feature_service,
                 tx_storage=tx_storage,
                 support_features=self._support_features,
                 not_support_features=self._not_support_features,
+                feature_storage=feature_storage,
             )
 
         return self._bit_signaling_service
@@ -477,6 +497,15 @@ def _get_or_create_verification_service(self) -> VerificationService:
 
         return self._verification_service
 
+    def _get_or_create_feature_storage(self) -> FeatureActivationStorage | None:
+        match self._storage_type:
+            case StorageType.MEMORY: return None
+            case StorageType.ROCKSDB: return FeatureActivationStorage(
+                settings=self._get_or_create_settings(),
+                rocksdb_storage=self._get_or_create_rocksdb_storage()
+            )
+            case _: assert_never(self._storage_type)
+
     def _get_or_create_vertex_verifiers(self) -> VertexVerifiers:
         if self._vertex_verifiers is None:
             settings = self._get_or_create_settings()

hathor/builder/cli_builder.py

@@ -13,7 +13,6 @@
 # limitations under the License.
 
 import getpass
-import json
 import os
 import platform
 import sys
@@ -27,8 +26,10 @@
 from hathor.daa import DifficultyAdjustmentAlgorithm
 from hathor.event import EventManager
 from hathor.exception import BuilderError
+from hathor.execution_manager import ExecutionManager
 from hathor.feature_activation.bit_signaling_service import BitSignalingService
 from hathor.feature_activation.feature_service import FeatureService
+from hathor.feature_activation.storage.feature_activation_storage import FeatureActivationStorage
 from hathor.indexes import IndexesManager, MemoryIndexesManager, RocksDBIndexesManager
 from hathor.manager import HathorManager
 from hathor.mining.cpu_mining_service import CpuMiningService
@@ -94,8 +95,7 @@ def create_manager(self, reactor: Reactor) -> HathorManager:
         self.log = logger.new()
         self.reactor = reactor
 
-        peer_id = self.create_peer_id()
-
+        peer_id = PeerId.create_from_json_path(self._args.peer) if self._args.peer else PeerId()
         python = f'{platform.python_version()}-{platform.python_implementation()}'
 
         self.log.info(
@@ -119,6 +119,7 @@ def create_manager(self, reactor: Reactor) -> HathorManager:
         tx_storage: TransactionStorage
         event_storage: EventStorage
         indexes: IndexesManager
+        feature_storage: FeatureActivationStorage | None = None
         self.rocksdb_storage: Optional[RocksDBStorage] = None
         self.event_ws_factory: Optional[EventWebsocketFactory] = None
 
@@ -151,6 +152,7 @@ def create_manager(self, reactor: Reactor) -> HathorManager:
                 kwargs['indexes'] = indexes
             tx_storage = TransactionRocksDBStorage(self.rocksdb_storage, **kwargs)
             event_storage = EventRocksDBStorage(self.rocksdb_storage)
+            feature_storage = FeatureActivationStorage(settings=settings, rocksdb_storage=self.rocksdb_storage)
 
         self.log.info('with storage', storage_class=type(tx_storage).__name__, path=self._args.data)
         if self._args.cache:
@@ -212,11 +214,14 @@ def create_manager(self, reactor: Reactor) -> HathorManager:
                 event_storage=event_storage
             )
 
+        execution_manager = ExecutionManager(reactor)
+
         event_manager = EventManager(
             event_storage=event_storage,
             event_ws_factory=self.event_ws_factory,
             pubsub=pubsub,
-            reactor=reactor
+            reactor=reactor,
+            execution_manager=execution_manager,
         )
 
         if self._args.wallet_index and tx_storage.indexes is not None:
@@ -236,7 +241,11 @@ def create_manager(self, reactor: Reactor) -> HathorManager:
             full_verification = True
 
         soft_voided_tx_ids = set(settings.SOFT_VOIDED_TX_IDS)
-        consensus_algorithm = ConsensusAlgorithm(soft_voided_tx_ids, pubsub=pubsub)
+        consensus_algorithm = ConsensusAlgorithm(
+            soft_voided_tx_ids,
+            pubsub=pubsub,
+            execution_manager=execution_manager
+        )
 
         if self._args.x_enable_event_queue:
             self.log.info('--x-enable-event-queue flag provided. '
@@ -252,7 +261,8 @@ def create_manager(self, reactor: Reactor) -> HathorManager:
             feature_service=self.feature_service,
             tx_storage=tx_storage,
             support_features=self._args.signal_support,
-            not_support_features=self._args.signal_not_support
+            not_support_features=self._args.signal_not_support,
+            feature_storage=feature_storage,
         )
 
         test_mode = TestMode.DISABLED
@@ -308,7 +318,8 @@ def create_manager(self, reactor: Reactor) -> HathorManager:
             feature_service=self.feature_service,
             bit_signaling_service=bit_signaling_service,
             verification_service=verification_service,
-            cpu_mining_service=cpu_mining_service
+            cpu_mining_service=cpu_mining_service,
+            execution_manager=execution_manager,
         )
 
         if self._args.x_ipython_kernel:
@@ -354,7 +365,7 @@ def create_manager(self, reactor: Reactor) -> HathorManager:
             self.log.warn('--memory-indexes is implied for memory storage or JSON storage')
 
         for description in self._args.listen:
-            p2p_manager.add_listen_address(description)
+            p2p_manager.add_listen_address_description(description)
 
         if self._args.peer_id_blacklist:
             self.log.info('with peer id blacklist', blacklist=self._args.peer_id_blacklist)
@@ -384,14 +395,6 @@ def get_hostname(self) -> Optional[str]:
             print('Hostname discovered and set to {}'.format(hostname))
         return hostname
 
-    def create_peer_id(self) -> PeerId:
-        if not self._args.peer:
-            peer_id = PeerId()
-        else:
-            data = json.load(open(self._args.peer, 'r'))
-            peer_id = PeerId.create_from_json(data)
-        return peer_id
-
     def create_wallet(self) -> BaseWallet:
         if self._args.wallet == 'hd':
             kwargs: dict[str, Any] = {

hathor/builder/sysctl_builder.py

@@ -13,7 +13,13 @@
 # limitations under the License.
 
 from hathor.builder import BuildArtifacts
-from hathor.sysctl import ConnectionsManagerSysctl, HathorManagerSysctl, Sysctl, WebsocketManagerSysctl
+from hathor.sysctl import (
+    ConnectionsManagerSysctl,
+    FeatureActivationSysctl,
+    HathorManagerSysctl,
+    Sysctl,
+    WebsocketManagerSysctl,
+)
 
 
 class SysctlBuilder:
@@ -25,7 +31,11 @@ def __init__(self, artifacts: BuildArtifacts) -> None:
     def build(self) -> Sysctl:
         """Build the sysctl tree."""
         root = Sysctl()
-        root.put_child('core', HathorManagerSysctl(self.artifacts.manager))
+
+        core = HathorManagerSysctl(self.artifacts.manager)
+        core.put_child('features', FeatureActivationSysctl(self.artifacts.bit_signaling_service))
+
+        root.put_child('core', core)
         root.put_child('p2p', ConnectionsManagerSysctl(self.artifacts.p2p_manager))
 
         ws_factory = self.artifacts.manager.metrics.websocket_factory

hathor/cli/main.py

@@ -49,6 +49,7 @@ def __init__(self) -> None:
             quick_test,
             replay_logs,
             reset_event_queue,
+            reset_feature_settings,
             run_node,
             shell,
             stratum_mining,
@@ -81,6 +82,8 @@ def __init__(self) -> None:
         self.add_cmd('oracle', 'oracle-encode-data', oracle_encode_data, 'Encode data and sign it with a private key')
         self.add_cmd('events', 'reset-event-queue', reset_event_queue, 'Delete all events and related data from the '
                                                                        'database')
+        self.add_cmd('features', 'reset-feature-settings', reset_feature_settings, 'Delete existing Feature '
+                     'Activation settings from the database')
         self.add_cmd('dev', 'shell', shell, 'Run a Python shell')
         self.add_cmd('dev', 'quick_test', quick_test, 'Similar to run_node but will quit after receiving a tx')
         self.add_cmd('dev', 'generate_nginx_config', nginx_config, 'Generate nginx config from OpenAPI json')

hathor/cli/mining.py

@@ -139,7 +139,8 @@ def execute(args: Namespace) -> None:
 
             from hathor.conf.get_settings import get_global_settings
             from hathor.daa import DifficultyAdjustmentAlgorithm
-            from hathor.verification.verification_service import VerificationService, VertexVerifiers
+            from hathor.verification.verification_service import VerificationService
+            from hathor.verification.vertex_verifiers import VertexVerifiers
             settings = get_global_settings()
             daa = DifficultyAdjustmentAlgorithm(settings=settings)
             verifiers = VertexVerifiers.create_defaults(settings=settings, daa=daa, feature_service=Mock())

hathor/cli/openapi_files/openapi_base.json

@@ -7,7 +7,7 @@
         ],
 	"info": {
             "title": "Hathor API",
-            "version": "0.59.0"
+            "version": "0.60.0"
 	},
 	"consumes": [
             "application/json"

hathor/cli/reset_feature_settings.py

@@ -0,0 +1,49 @@
+#  Copyright 2023 Hathor Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+from argparse import ArgumentParser, Namespace
+
+from structlog import get_logger
+
+logger = get_logger()
+
+
+def create_parser() -> ArgumentParser:
+    from hathor.cli.util import create_parser
+
+    parser = create_parser()
+    parser.add_argument('--data', help='Data directory')
+
+    return parser
+
+
+def execute(args: Namespace) -> None:
+    from hathor.conf.get_settings import get_global_settings
+    from hathor.feature_activation.storage.feature_activation_storage import FeatureActivationStorage
+    from hathor.storage import RocksDBStorage
+
+    assert args.data is not None, '--data is required'
+
+    rocksdb_storage = RocksDBStorage(path=args.data)
+    feature_storage = FeatureActivationStorage(settings=get_global_settings(), rocksdb_storage=rocksdb_storage)
+
+    logger.info('removing feature activation settings...')
+    feature_storage.reset_settings()
+    logger.info('reset complete')
+
+
+def main():
+    parser = create_parser()
+    args = parser.parse_args()
+    execute(args)

hathor/cli/run_node.py

@@ -221,7 +221,8 @@ def prepare(self, *, register_resources: bool = True) -> None:
             wallet=self.manager.wallet,
             rocksdb_storage=getattr(builder, 'rocksdb_storage', None),
             stratum_factory=self.manager.stratum_factory,
-            feature_service=self.manager._feature_service
+            feature_service=self.manager._feature_service,
+            bit_signaling_service=self.manager._bit_signaling_service,
         )
 
     def start_sentry_if_possible(self) -> None:
@@ -264,9 +265,8 @@ def register_signal_handlers(self) -> None:
     def signal_usr1_handler(self, sig: int, frame: Any) -> None:
         """Called when USR1 signal is received."""
         try:
-            self.log.warn('USR1 received. Killing all connections...')
-            if self.manager and self.manager.connections:
-                self.manager.connections.disconnect_all_peers(force=True)
+            self.log.warn('USR1 received.')
+            self.manager.connections.reload_entrypoints_and_connections()
         except Exception:
             # see: https://docs.python.org/3/library/signal.html#note-on-signal-handlers-and-exceptions
             self.log.error('prevented exception from escaping the signal handler', exc_info=True)

hathor/conf/settings.py

@@ -423,6 +423,10 @@ def GENESIS_TX2_TIMESTAMP(self) -> int:
     OLD_MAX_MERKLE_PATH_LENGTH: int = 12
     NEW_MAX_MERKLE_PATH_LENGTH: int = 20
 
+    # Maximum number of tx tips to accept in the initial phase of the mempool sync 1000 is arbitrary, but it should be
+    # more than enough for the forseeable future
+    MAX_MEMPOOL_RECEIVING_TIPS: int = 1000
+
     # Used to enable nano contracts.
     #
     # This should NEVER be enabled for mainnet and testnet, since both networks will