Merge pull request #665 from HathorNetwork/refactor/use-none-height-m…

…etadata

refactor: use None by default for height metadata

hathor/cli/db_export.py

@@ -95,6 +95,7 @@ def iter_tx(self) -> Iterator['BaseTransaction']:
             yield tx
 
     def run(self) -> None:
+        from hathor.transaction import Block
         from hathor.util import tx_progress
         self.log.info('export')
         self.out_file.write(MAGIC_HEADER)
@@ -112,9 +113,10 @@ def run(self) -> None:
             assert tx.hash is not None
             tx_meta = tx.get_metadata()
             if tx.is_block:
+                assert isinstance(tx, Block)
                 if not tx_meta.voided_by:
                     # XXX: max() shouldn't be needed, but just in case
-                    best_height = max(best_height, tx_meta.height)
+                    best_height = max(best_height, tx.get_height())
                 block_count += 1
             else:
                 tx_count += 1

hathor/consensus/block_consensus.py

@@ -146,7 +146,7 @@ def update_voided_info(self, block: Block) -> None:
             # we need to check that block is not voided.
             meta = block.get_metadata()
             if not meta.voided_by:
-                storage.indexes.height.add_new(meta.height, block.hash, block.timestamp)
+                storage.indexes.height.add_new(block.get_height(), block.hash, block.timestamp)
                 storage.update_best_block_tips_cache([block.hash])
             # The following assert must be true, but it is commented out for performance reasons.
             if settings.SLOW_ASSERTS:
@@ -206,10 +206,11 @@ def update_voided_info(self, block: Block) -> None:
                     # As `update_score_and_mark_as_the_best_chain_if_possible` may affect `voided_by`,
                     # we need to check that block is not voided.
                     meta = block.get_metadata()
+                    height = block.get_height()
                     if not meta.voided_by:
-                        self.log.debug('index new winner block', height=meta.height, block=block.hash_hex)
+                        self.log.debug('index new winner block', height=height, block=block.hash_hex)
                         # We update the height cache index with the new winner chain
-                        storage.indexes.height.update_new_chain(meta.height, block)
+                        storage.indexes.height.update_new_chain(height, block)
                         storage.update_best_block_tips_cache([block.hash])
                         # It is only a re-org if common_block not in heads
                         if common_block not in heads:

hathor/consensus/consensus.py

@@ -125,10 +125,9 @@ def _unsafe_update(self, base: BaseTransaction) -> None:
         # emit the reorg started event if needed
         if context.reorg_common_block is not None:
             old_best_block = base.storage.get_transaction(best_tip)
+            assert isinstance(old_best_block, Block)
             new_best_block = base.storage.get_transaction(new_best_tip)
-            old_best_block_meta = old_best_block.get_metadata()
-            common_block_meta = context.reorg_common_block.get_metadata()
-            reorg_size = old_best_block_meta.height - common_block_meta.height
+            reorg_size = old_best_block.get_height() - context.reorg_common_block.get_height()
             assert old_best_block != new_best_block
             assert reorg_size > 0
             context.pubsub.publish(HathorEvents.REORG_STARTED, old_best_height=best_height,

hathor/daa.py

@@ -83,7 +83,7 @@ def calculate_next_weight(parent_block: 'Block', timestamp: int) -> float:
     from hathor.transaction import sum_weights
 
     root = parent_block
-    N = min(2 * settings.BLOCK_DIFFICULTY_N_BLOCKS, parent_block.get_metadata().height - 1)
+    N = min(2 * settings.BLOCK_DIFFICULTY_N_BLOCKS, parent_block.get_height() - 1)
     K = N // 2
     T = AVG_TIME_BETWEEN_BLOCKS
     S = 5

hathor/indexes/deps_index.py

@@ -45,7 +45,8 @@ def get_requested_from_height(tx: BaseTransaction) -> int:
     """
     assert tx.storage is not None
     if tx.is_block:
-        return tx.get_metadata().height
+        assert isinstance(tx, Block)
+        return tx.get_height()
     first_block = tx.get_metadata().first_block
     if first_block is None:
         # XXX: consensus did not run yet to update first_block, what should we do?
@@ -54,7 +55,7 @@ def get_requested_from_height(tx: BaseTransaction) -> int:
         return INF_HEIGHT
     block = tx.storage.get_transaction(first_block)
     assert isinstance(block, Block)
-    return block.get_metadata().height
+    return block.get_height()
 
 
 class DepsIndex(BaseIndex):

hathor/indexes/height_index.py

@@ -57,10 +57,9 @@ def init_loop_step(self, tx: BaseTransaction) -> None:
             return
         assert isinstance(tx, Block)
         assert tx.hash is not None
-        tx_meta = tx.get_metadata()
-        if tx_meta.voided_by:
+        if tx.get_metadata().voided_by:
             return
-        self.add_new(tx_meta.height, tx.hash, tx.timestamp)
+        self.add_new(tx.get_height(), tx.hash, tx.timestamp)
 
     @abstractmethod
     def add_new(self, height: int, block_hash: bytes, timestamp: int) -> None:
@@ -105,7 +104,7 @@ def update_new_chain(self, height: int, block: Block) -> None:
             )
 
             side_chain_block = side_chain_block.get_block_parent()
-            new_block_height = side_chain_block.get_metadata().height
+            new_block_height = side_chain_block.get_height()
             assert new_block_height + 1 == block_height
             block_height = new_block_height
 

hathor/indexes/utxo_index.py

@@ -21,7 +21,7 @@
 from hathor.conf import HathorSettings
 from hathor.indexes.base_index import BaseIndex
 from hathor.indexes.scope import Scope
-from hathor.transaction import BaseTransaction, TxOutput
+from hathor.transaction import BaseTransaction, Block, TxOutput
 from hathor.transaction.scripts import parse_address_script
 from hathor.util import sorted_merger
 
@@ -69,9 +69,11 @@ def from_tx_output(cls, tx: BaseTransaction, index: int, tx_output: TxOutput) ->
         if address_script is None:
             raise ValueError('UtxoIndexItem can only be used with scripts supported by `parse_address_script`')
 
-        tx_meta = tx.get_metadata()
-
-        heightlock: Optional[int] = tx_meta.height + settings.REWARD_SPEND_MIN_BLOCKS if tx.is_block else None
+        heightlock: Optional[int]
+        if isinstance(tx, Block):
+            heightlock = tx.get_height() + settings.REWARD_SPEND_MIN_BLOCKS
+        else:
+            heightlock = None
         # XXX: timelock forced to None when there is a heightlock
         timelock: Optional[int] = address_script.get_timelock() if heightlock is None else None
         # XXX: that is, at least one of them must but None

hathor/manager.py

@@ -409,7 +409,7 @@ def _initialize_components_full_verification(self) -> None:
             dt = LogDuration(t2 - t1)
             dcnt = cnt - cnt2
             tx_rate = '?' if dt == 0 else dcnt / dt
-            h = max(h, tx_meta.height)
+            h = max(h, tx_meta.height or 0)
             if dt > 30:
                 ts_date = datetime.datetime.fromtimestamp(self.tx_storage.latest_timestamp)
                 if h == 0:
@@ -453,14 +453,16 @@ def _initialize_components_full_verification(self) -> None:
                 # this works because blocks on the best chain are iterated from lower to higher height
                 assert tx.hash is not None
                 assert tx_meta.validation.is_at_least_basic()
+                assert isinstance(tx, Block)
+                blk_height = tx.get_height()
                 if not tx_meta.voided_by and tx_meta.validation.is_fully_connected():
                     # XXX: this might not be needed when making a full init because the consensus should already have
-                    self.tx_storage.indexes.height.add_reorg(tx_meta.height, tx.hash, tx.timestamp)
+                    self.tx_storage.indexes.height.add_reorg(blk_height, tx.hash, tx.timestamp)
 
                 # Check if it's a checkpoint block
-                if tx_meta.height in checkpoint_heights:
-                    if tx.hash == checkpoint_heights[tx_meta.height]:
-                        del checkpoint_heights[tx_meta.height]
+                if blk_height in checkpoint_heights:
+                    if tx.hash == checkpoint_heights[blk_height]:
+                        del checkpoint_heights[blk_height]
                     else:
                         # If the hash is different from checkpoint hash, we stop the node
                         self.log.error('Error initializing the node. Checkpoint validation error.')
@@ -794,7 +796,7 @@ def _make_block_template(self, parent_block: Block, parent_txs: 'ParentTxs', cur
         # protect agains a weight that is too small but using WEIGHT_TOL instead of 2*WEIGHT_TOL)
         min_significant_weight = calculate_min_significant_weight(parent_block_metadata.score, 2 * settings.WEIGHT_TOL)
         weight = max(daa.calculate_next_weight(parent_block, timestamp), min_significant_weight)
-        height = parent_block_metadata.height + 1
+        height = parent_block.get_height() + 1
         parents = [parent_block.hash] + parent_txs.must_include
         parents_any = parent_txs.can_include
         # simplify representation when you only have one to choose from

hathor/transaction/block.py

@@ -103,7 +103,7 @@ def calculate_height(self) -> int:
             return 0
         assert self.storage is not None
         parent_block = self.get_block_parent()
-        return parent_block.get_metadata().height + 1
+        return parent_block.get_height() + 1
 
     def calculate_min_height(self) -> int:
         """The minimum height the next block needs to have, basically the maximum min-height of this block's parents.
@@ -301,16 +301,18 @@ def verify_basic(self, skip_block_weight_verification: bool = False) -> None:
     def verify_checkpoint(self, checkpoints: list[Checkpoint]) -> None:
         assert self.hash is not None
         assert self.storage is not None
-        meta = self.get_metadata()
-        # XXX: it's fine to use `in` with NamedTuples
-        if Checkpoint(meta.height, self.hash) in checkpoints:
-            return
-        # otherwise at least one child must be checkpoint validated
-        for child_tx in map(self.storage.get_transaction, meta.children):
-            if child_tx.get_metadata().validation.is_checkpoint():
-                return
-        raise CheckpointError(f'Invalid new block {self.hash_hex}: expected to reach a checkpoint but none of '
-                              'its children is checkpoint-valid and its hash does not match any checkpoint')
+        height = self.get_height()  # TODO: use "soft height" when sync-checkpoint is added
+        # find checkpoint with our height:
+        checkpoint: Optional[Checkpoint] = None
+        for cp in checkpoints:
+            if cp.height == height:
+                checkpoint = cp
+                break
+        if checkpoint is not None and checkpoint.hash != self.hash:
+            raise CheckpointError(f'Invalid new block {self.hash_hex}: checkpoint hash does not match')
+        else:
+            # TODO: check whether self is a parent of any checkpoint-valid block, this is left for a future PR
+            raise NotImplementedError
 
     def verify_weight(self) -> None:
         """Validate minimum block difficulty."""
@@ -322,13 +324,14 @@ def verify_weight(self) -> None:
     def verify_height(self) -> None:
         """Validate that the block height is enough to confirm all transactions being confirmed."""
         meta = self.get_metadata()
+        assert meta.height is not None
         if meta.height < meta.min_height:
             raise RewardLocked(f'Block needs {meta.min_height} height but has {meta.height}')
 
     def verify_reward(self) -> None:
         """Validate reward amount."""
         parent_block = self.get_block_parent()
-        tokens_issued_per_block = daa.get_tokens_issued_per_block(parent_block.get_metadata().height + 1)
+        tokens_issued_per_block = daa.get_tokens_issued_per_block(parent_block.get_height() + 1)
         if self.sum_outputs != tokens_issued_per_block:
             raise InvalidBlockReward(
                 f'Invalid number of issued tokens tag=invalid_issued_tokens tx.hash={self.hash_hex} '
@@ -386,7 +389,9 @@ def verify(self, reject_locked_reward: bool = True) -> None:
 
     def get_height(self) -> int:
         """Returns the block's height."""
-        return self.get_metadata().height
+        meta = self.get_metadata()
+        assert meta.height is not None
+        return meta.height
 
     def get_feature_activation_bit_counts(self) -> list[int]:
         """Returns the block's feature_activation_bit_counts metadata attribute."""

hathor/transaction/storage/transaction_storage.py

@@ -646,7 +646,8 @@ def get_height_best_block(self) -> int:
         heads = [self.get_transaction(h) for h in self.get_best_block_tips()]
         highest_height = 0
         for head in heads:
-            head_height = head.get_metadata().height
+            assert isinstance(head, Block)
+            head_height = head.get_height()
             if head_height > highest_height:
                 highest_height = head_height
 

hathor/transaction/transaction.py

@@ -154,7 +154,7 @@ def _calculate_my_min_height(self) -> int:
         """ Calculates min height derived from own spent rewards"""
         min_height = 0
         for blk in self.iter_spent_rewards():
-            min_height = max(min_height, blk.get_metadata().height + settings.REWARD_SPEND_MIN_BLOCKS + 1)
+            min_height = max(min_height, blk.get_height() + settings.REWARD_SPEND_MIN_BLOCKS + 1)
         return min_height
 
     def get_funds_fields_from_struct(self, buf: bytes, *, verbose: VerboseCallback = None) -> bytes:
@@ -593,12 +593,18 @@ def get_spent_reward_locked_info(self) -> Optional[RewardLockedInfo]:
 
     def _spent_reward_needed_height(self, block: Block) -> int:
         """ Returns height still needed to unlock this reward: 0 means it's unlocked."""
+        import math
         assert self.storage is not None
         # omitting timestamp to get the current best block, this will usually hit the cache instead of being slow
         tips = self.storage.get_best_block_tips()
         assert len(tips) > 0
-        best_height = min(self.storage.get_transaction(tip).get_metadata().height for tip in tips)
-        spent_height = block.get_metadata().height
+        best_height = math.inf
+        for tip in tips:
+            blk = self.storage.get_transaction(tip)
+            assert isinstance(blk, Block)
+            best_height = min(best_height, blk.get_height())
+        assert isinstance(best_height, int)
+        spent_height = block.get_height()
         spend_blocks = best_height - spent_height
         needed_height = settings.REWARD_SPEND_MIN_BLOCKS - spend_blocks
         return max(needed_height, 0)

hathor/transaction/transaction_metadata.py

@@ -37,7 +37,7 @@ class TransactionMetadata:
     accumulated_weight: float
     score: float
     first_block: Optional[bytes]
-    height: int
+    height: Optional[int]
     validation: ValidationState
     # XXX: this is only used to defer the reward-lock verification from the transaction spending a reward to the first
     # block that confirming this transaction, it is important to always have this set to be able to distinguish an old
@@ -62,7 +62,7 @@ def __init__(
         hash: Optional[bytes] = None,
         accumulated_weight: float = 0,
         score: float = 0,
-        height: int = 0,
+        height: Optional[int] = None,
         min_height: int = 0,
         feature_activation_bit_counts: Optional[list[int]] = None
     ) -> None:

hathor/util.py

@@ -520,8 +520,10 @@ def _tx_progress(iter_tx: Iterator['BaseTransaction'], *, log: 'structlog.stdlib
             log.warn('iterator was slow to yield', took_sec=dt_next)
 
         assert tx.hash is not None
-        tx_meta = tx.get_metadata()
-        h = max(h, tx_meta.height)
+        # XXX: this is only informative and made to work with either partially/fully validated blocks/transactions
+        meta = tx.get_metadata()
+        if meta.height:
+            h = max(h, meta.height)
         ts_tx = max(ts_tx, tx.timestamp)
 
         t_log = time.time()

hathor/wallet/base_wallet.py

@@ -27,7 +27,7 @@
 from hathor.conf import HathorSettings
 from hathor.crypto.util import decode_address
 from hathor.pubsub import EventArguments, HathorEvents, PubSubManager
-from hathor.transaction import BaseTransaction, TxInput, TxOutput
+from hathor.transaction import BaseTransaction, Block, TxInput, TxOutput
 from hathor.transaction.base_transaction import int_to_bytes
 from hathor.transaction.scripts import P2PKH, create_output_script, parse_address_script
 from hathor.transaction.storage import TransactionStorage
@@ -499,7 +499,8 @@ def get_inputs_from_amount(
     def can_spend_block(self, tx_storage: 'TransactionStorage', tx_id: bytes) -> bool:
         tx = tx_storage.get_transaction(tx_id)
         if tx.is_block:
-            if tx_storage.get_height_best_block() - tx.get_metadata().height < settings.REWARD_SPEND_MIN_BLOCKS:
+            assert isinstance(tx, Block)
+            if tx_storage.get_height_best_block() - tx.get_height() < settings.REWARD_SPEND_MIN_BLOCKS:
                 return False
         return True