Merge pull request #910 from HathorNetwork/feat/new-max-merkle-path-l…

…ength

feat(merged-mining): configure new max merkle path length on testnet

hathor/conf/settings.py

@@ -419,6 +419,10 @@ def GENESIS_TX2_TIMESTAMP(self) -> int:
     # Time in seconds to request the best blockchain from peers.
     BEST_BLOCKCHAIN_INTERVAL: int = 5  # seconds
 
+    # Merged mining settings. The old value is going to be replaced by the new value through Feature Activation.
+    OLD_MAX_MERKLE_PATH_LENGTH: int = 12
+    NEW_MAX_MERKLE_PATH_LENGTH: int = 20
+
     @classmethod
     def from_yaml(cls, *, filepath: str) -> 'HathorSettings':
         """Takes a filepath to a yaml file and returns a validated HathorSettings instance."""

hathor/conf/testnet.py

@@ -14,6 +14,8 @@
 
 from hathor.checkpoint import Checkpoint as cp
 from hathor.conf.settings import HathorSettings
+from hathor.feature_activation.feature import Feature
+from hathor.feature_activation.model.criteria import Criteria
 from hathor.feature_activation.settings import Settings as FeatureActivationSettings
 
 SETTINGS = HathorSettings(
@@ -54,5 +56,19 @@
     ],
     FEATURE_ACTIVATION=FeatureActivationSettings(
         default_threshold=15_120,  # 15120 = 75% of evaluation_interval (20160)
+        features={
+            Feature.INCREASE_MAX_MERKLE_PATH_LENGTH: Criteria(
+                bit=3,
+                # N = 3_548_160
+                # Expected to be reached around Sunday, 2024-02-04.
+                # Right now the best block is 3_521_000 on testnet (2024-01-26).
+                start_height=3_548_160,
+                timeout_height=3_588_480,  # N + 2 * 20160 (2 weeks after the start)
+                minimum_activation_height=0,
+                lock_in_on_timeout=False,
+                version='0.59.0',
+                signal_support_by_default=True,
+            )
+        }
     )
 )

hathor/conf/testnet.yml

@@ -38,3 +38,15 @@ CHECKPOINTS:
 
 FEATURE_ACTIVATION:
   default_threshold: 15_120 # 15120 = 75% of evaluation_interval (20160)
+  features:
+    INCREASE_MAX_MERKLE_PATH_LENGTH:
+      bit: 3
+      # N = 3_548_160
+      # Expected to be reached around Sunday, 2024-02-04.
+      # Right now the best block is 3_521_000 on testnet (2024-01-26).
+      start_height: 3_548_160
+      timeout_height: 3_588_480
+      minimum_activation_height: 0
+      lock_in_on_timeout: false
+      version: 0.59.0
+      signal_support_by_default: true

hathor/feature_activation/feature.py

@@ -33,3 +33,5 @@ class Feature(Enum):
     NOP_FEATURE_4 = 'NOP_FEATURE_4'
     NOP_FEATURE_5 = 'NOP_FEATURE_5'
     NOP_FEATURE_6 = 'NOP_FEATURE_6'
+
+    INCREASE_MAX_MERKLE_PATH_LENGTH = 'INCREASE_MAX_MERKLE_PATH_LENGTH'

hathor/merged_mining/coordinator.py

@@ -624,7 +624,7 @@ def handle_submit(self, params: list[Any], msgid: Optional[str]) -> None:
 
         try:
             aux_pow = job.build_aux_pow(work)
-            aux_pow.verify(block_base_hash)
+            aux_pow.verify_magic_number(block_base_hash)
         except TxValidationError as e:
             self.log.warn('invalid work', job_id=work.job_id, error=e)
             self.send_error(INVALID_SOLUTION, data={'message': 'Job has invalid work.'})

hathor/transaction/aux_pow.py

@@ -19,9 +19,6 @@
 logger = get_logger()
 
 
-MAX_MERKLE_PATH_LENGTH: int = 12
-
-
 class BitcoinAuxPow(NamedTuple):
     header_head: bytes  # 36 bytes
     coinbase_head: bytes  # variable length (at least 47 bytes)
@@ -44,23 +41,28 @@ def calculate_hash(self, base_block_hash: bytes) -> bytes:
         merkle_root = bytes(reversed(build_merkle_root_from_path([coinbase_tx_hash] + self.merkle_path)))
         return sha256d_hash(self.header_head + merkle_root + self.header_tail)
 
-    def verify(self, _base_block_hash: bytes) -> None:
+    def verify(self, _base_block_hash: bytes, max_merkle_path_length: int) -> None:
         """ Check for inconsistencies, raises instance of TxValidationError on error.
         """
+        self.verify_magic_number(_base_block_hash)
+        self.verify_merkle_path(_base_block_hash, max_merkle_path_length)
+
+    def verify_magic_number(self, _base_block_hash: bytes) -> None:
+        """Check that the `MAGIC_NUMBER` is present and in the correct index."""
         from hathor.merged_mining import MAGIC_NUMBER
-        from hathor.transaction.exceptions import (
-            AuxPowLongMerklePathError,
-            AuxPowNoMagicError,
-            AuxPowUnexpectedMagicError,
-        )
+        from hathor.transaction.exceptions import AuxPowNoMagicError, AuxPowUnexpectedMagicError
         magic_index = self.coinbase_head.find(MAGIC_NUMBER)
         if magic_index == -1:
             raise AuxPowNoMagicError('cannot find MAGIC_NUMBER')
         if magic_index < len(self.coinbase_head) - len(MAGIC_NUMBER):
             raise AuxPowUnexpectedMagicError('unexpected MAGIC_NUMBER')
+
+    def verify_merkle_path(self, _base_block_hash: bytes, max_merkle_path_length: int) -> None:
+        """Check that the merkle path length is smaller than the maximum limit."""
+        from hathor.transaction.exceptions import AuxPowLongMerklePathError
         merkle_path_length = len(self.merkle_path)
-        if merkle_path_length > MAX_MERKLE_PATH_LENGTH:
-            raise AuxPowLongMerklePathError(f'merkle_path too long: {merkle_path_length} > {MAX_MERKLE_PATH_LENGTH}')
+        if merkle_path_length > max_merkle_path_length:
+            raise AuxPowLongMerklePathError(f'merkle_path too long: {merkle_path_length} > {max_merkle_path_length}')
 
     def __bytes__(self) -> bytes:
         """ Convert to byte representation.

hathor/verification/merge_mined_block_verifier.py

@@ -12,14 +12,31 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
+from hathor.conf.settings import HathorSettings
+from hathor.feature_activation.feature import Feature
+from hathor.feature_activation.feature_service import FeatureService
 from hathor.transaction import MergeMinedBlock
 
 
 class MergeMinedBlockVerifier:
-    __slots__ = ()
+    __slots__ = ('_settings', '_feature_service',)
+
+    def __init__(self, *, settings: HathorSettings, feature_service: FeatureService):
+        self._settings = settings
+        self._feature_service = feature_service
 
     def verify_aux_pow(self, block: MergeMinedBlock) -> None:
         """ Verify auxiliary proof-of-work (for merged mining).
         """
         assert block.aux_pow is not None
-        block.aux_pow.verify(block.get_base_hash())
+
+        is_feature_active = self._feature_service.is_feature_active(
+            block=block,
+            feature=Feature.INCREASE_MAX_MERKLE_PATH_LENGTH
+        )
+        max_merkle_path_length = (
+            self._settings.NEW_MAX_MERKLE_PATH_LENGTH if is_feature_active
+            else self._settings.OLD_MAX_MERKLE_PATH_LENGTH
+        )
+
+        block.aux_pow.verify(block.get_base_hash(), max_merkle_path_length)

hathor/verification/vertex_verifiers.py

@@ -66,7 +66,7 @@ def create(
         Create a VertexVerifiers instance using a custom vertex_verifier.
         """
         block_verifier = BlockVerifier(settings=settings, daa=daa, feature_service=feature_service)
-        merge_mined_block_verifier = MergeMinedBlockVerifier()
+        merge_mined_block_verifier = MergeMinedBlockVerifier(settings=settings, feature_service=feature_service)
         tx_verifier = TransactionVerifier(settings=settings, daa=daa)
         token_creation_tx_verifier = TokenCreationTransactionVerifier(settings=settings)
 

tests/tx/test_tx.py

@@ -1,9 +1,12 @@
 import base64
 import hashlib
 from math import isinf, isnan
+from unittest.mock import patch
 
 from hathor.crypto.util import decode_address, get_address_from_public_key, get_private_key_from_bytes
 from hathor.daa import TestMode
+from hathor.feature_activation.feature import Feature
+from hathor.feature_activation.feature_service import FeatureService
 from hathor.simulator.utils import add_new_blocks
 from hathor.transaction import MAX_OUTPUT_VALUE, Block, Transaction, TxInput, TxOutput
 from hathor.transaction.exceptions import (
@@ -222,15 +225,19 @@ def test_merge_mined_no_magic(self):
         from hathor.transaction.exceptions import AuxPowNoMagicError
         from hathor.transaction.merge_mined_block import MergeMinedBlock
 
-        parents = [tx.hash for tx in self.genesis]
+        parent_block = self.genesis_blocks[0].hash
+        parent_txs = [tx.hash for tx in self.genesis_txs]
+        parents = [parent_block, *parent_txs]
         address = decode_address(self.get_address(1))
         outputs = [TxOutput(100, P2PKH.create_output_script(address))]
 
         b = MergeMinedBlock(
+            hash=b'some_hash',
             timestamp=self.genesis_blocks[0].timestamp + 1,
             weight=1,
             outputs=outputs,
             parents=parents,
+            storage=self.tx_storage,
             aux_pow=BitcoinAuxPow(
                 b'\x00' * 32,
                 b'\x00' * 42,  # no MAGIC_NUMBER
@@ -253,25 +260,31 @@ def test_merge_mined_multiple_magic(self):
         from hathor.transaction.exceptions import AuxPowUnexpectedMagicError
         from hathor.transaction.merge_mined_block import MergeMinedBlock
 
-        parents = [tx.hash for tx in self.genesis]
+        parent_block = self.genesis_blocks[0].hash
+        parent_txs = [tx.hash for tx in self.genesis_txs]
+        parents = [parent_block, *parent_txs]
         address1 = decode_address(self.get_address(1))
         address2 = decode_address(self.get_address(2))
         assert address1 != address2
         outputs1 = [TxOutput(100, P2PKH.create_output_script(address1))]
         outputs2 = [TxOutput(100, P2PKH.create_output_script(address2))]
 
         b1 = MergeMinedBlock(
+            hash=b'some_hash1',
             timestamp=self.genesis_blocks[0].timestamp + 1,
             weight=1,
             outputs=outputs1,
             parents=parents,
+            storage=self.tx_storage,
         )
 
         b2 = MergeMinedBlock(
+            hash=b'some_hash2',
             timestamp=self.genesis_blocks[0].timestamp + 1,
             weight=1,
             outputs=outputs2,
             parents=parents,
+            storage=self.tx_storage,
         )
 
         assert b1.get_base_hash() != b2.get_base_hash()
@@ -321,6 +334,16 @@ def test_merge_mined_long_merkle_path(self):
         address = decode_address(self.get_address(1))
         outputs = [TxOutput(100, P2PKH.create_output_script(address))]
 
+        patch_path = 'hathor.feature_activation.feature_service.FeatureService.is_feature_active'
+
+        def is_feature_active_false(self: FeatureService, *, block: Block, feature: Feature) -> bool:
+            assert feature == Feature.INCREASE_MAX_MERKLE_PATH_LENGTH
+            return False
+
+        def is_feature_active_true(self: FeatureService, *, block: Block, feature: Feature) -> bool:
+            assert feature == Feature.INCREASE_MAX_MERKLE_PATH_LENGTH
+            return True
+
         b = MergeMinedBlock(
             timestamp=self.genesis_blocks[0].timestamp + 1,
             weight=1,
@@ -330,17 +353,42 @@ def test_merge_mined_long_merkle_path(self):
                 b'\x00' * 32,
                 b'\x00' * 42 + MAGIC_NUMBER,
                 b'\x00' * 18,
-                [b'\x00' * 32] * 13,  # 1 too long
+                [b'\x00' * 32] * (self._settings.OLD_MAX_MERKLE_PATH_LENGTH + 1),  # 1 too long
                 b'\x00' * 12,
             )
         )
 
-        with self.assertRaises(AuxPowLongMerklePathError):
+        # Test with the INCREASE_MAX_MERKLE_PATH_LENGTH feature disabled
+        with patch(patch_path, is_feature_active_false):
+            with self.assertRaises(AuxPowLongMerklePathError):
+                self._verifiers.merge_mined_block.verify_aux_pow(b)
+
+            # removing one path makes it work
+            b.aux_pow.merkle_path.pop()
             self._verifiers.merge_mined_block.verify_aux_pow(b)
 
-        # removing one path makes it work
-        b.aux_pow.merkle_path.pop()
-        self._verifiers.merge_mined_block.verify_aux_pow(b)
+        b2 = MergeMinedBlock(
+            timestamp=self.genesis_blocks[0].timestamp + 1,
+            weight=1,
+            outputs=outputs,
+            parents=parents,
+            aux_pow=BitcoinAuxPow(
+                b'\x00' * 32,
+                b'\x00' * 42 + MAGIC_NUMBER,
+                b'\x00' * 18,
+                [b'\x00' * 32] * (self._settings.NEW_MAX_MERKLE_PATH_LENGTH + 1),  # 1 too long
+                b'\x00' * 12,
+            )
+        )
+
+        # Test with the INCREASE_MAX_MERKLE_PATH_LENGTH feature enabled
+        with patch(patch_path, is_feature_active_true):
+            with self.assertRaises(AuxPowLongMerklePathError):
+                self._verifiers.merge_mined_block.verify_aux_pow(b2)
+
+            # removing one path makes it work
+            b2.aux_pow.merkle_path.pop()
+            self._verifiers.merge_mined_block.verify_aux_pow(b2)
 
     def test_block_outputs(self):
         from hathor.transaction.exceptions import TooManyOutputs