CMerkleBlock class and wire messages

bitcoin/core/__init__.py

@@ -539,6 +539,123 @@ def GetHash(self):
             object.__setattr__(self, '_cached_GetHash', _cached_GetHash)
             return _cached_GetHash
 
+class CMerkleBlock(CBlockHeader):
+    """
+    The merkle block returned to spv clients when a filter is set on the remote peer.
+    """
+
+    __slots__ = ['nTX', 'vHashes', 'vFlags']
+
+    def __init__(self, nVersion=3, hashPrevBlock=b'\x00'*32, hashMerkleRoot=b'\x00'*32, nTime=0, nBits=0, nNonce=0, nTX=0, vHashes=(), vFlags=()):
+        """Create a new block"""
+        super(CMerkleBlock, self).__init__(nVersion, hashPrevBlock, hashMerkleRoot, nTime, nBits, nNonce)
+
+        object.__setattr__(self, 'nTX', nTX)
+        object.__setattr__(self, 'vHashes', vHashes)
+        object.__setattr__(self, 'vFlags', vFlags)
+
+    @classmethod
+    def stream_deserialize(cls, f):
+
+        def bits(f, n):
+            ret = []
+            bytes = (ord(b) for b in f.read(n))
+            for b in bytes:
+                for i in xrange(8):
+                    ret.append((b >> i) & 1)
+            return ret
+
+        self = super(CMerkleBlock, cls).stream_deserialize(f)
+
+        nTX = struct.unpack('<L', ser_read(f, 4))[0]
+        nHashes = VarIntSerializer.stream_deserialize(f)
+        vHashes = []
+        for i in range(nHashes):
+            vHashes.append(ser_read(f, 32))
+        nFlags = VarIntSerializer.stream_deserialize(f)
+        vFlags = bits(f, nFlags)
+        object.__setattr__(self, 'nTX', nTX)
+        object.__setattr__(self, 'vHashes', vHashes)
+        object.__setattr__(self, 'vFlags', vFlags)
+
+        return self
+
+    def stream_serialize(self, f):
+        super(CMerkleBlock, self).stream_serialize(f)
+        f.write(struct.pack('<L', self.nTX))
+        VarIntSerializer.stream_serialize(len(self.vHashes), f)
+        for hash in self.vHashes:
+            f.write(hash)
+        VarIntSerializer.stream_serialize(int(len(self.vFlags)/8), f)
+        bin_string = ""
+        for bit in self.vFlags:
+            bin_string += str(bit)
+            if len(bin_string) == 8:
+                f.write(struct.pack('B', int(bin_string[::-1], 2)))
+                bin_string = ""
+
+    def get_matched_txs(self):
+        """
+        Return a list of transaction hashes that matched the filter. These txs
+        have been validated against the merkle tree structure and are definitely
+        in the block. However, the block hash still needs to be checked against
+        the best chain.
+        """
+        def getTreeWidth(transaction_count, height):
+            return (transaction_count + (1 << height) - 1) >> height
+
+        matched_hashes = []
+
+        def recursive_extract_hashes(height, pos):
+            parent_of_match = bool(self.vFlags.pop(0))
+            if height == 0 or not parent_of_match:
+                hash = self.vHashes.pop(0)
+                if height == 0 and parent_of_match:
+                    matched_hashes.append(hash)
+                return hash
+            else:
+                left = recursive_extract_hashes(height - 1, pos * 2)
+                if pos * 2 + 1 < getTreeWidth(self.nTX, height-1):
+                    right = recursive_extract_hashes(height - 1, pos * 2 + 1)
+                    if left == right:
+                        raise Exception("Invalid Merkle Tree")
+                else:
+                    right = left
+                return sha256(sha256(left+right).digest()).digest()
+
+        height = 0
+        while getTreeWidth(self.nTX, height) > 1:
+            height += 1
+
+        calculated_root = recursive_extract_hashes(height, 0)
+        if calculated_root == self.get_header().hashMerkleRoot:
+            return matched_hashes
+        else:
+            return None
+
+    def get_header(self):
+        """Return the block header
+        Returned header is a new object.
+        """
+        return CBlockHeader(nVersion=self.nVersion,
+                            hashPrevBlock=self.hashPrevBlock,
+                            hashMerkleRoot=self.hashMerkleRoot,
+                            nTime=self.nTime,
+                            nBits=self.nBits,
+                            nNonce=self.nNonce)
+
+    def GetHash(self):
+        """Return the block hash
+        Note that this is the hash of the header, not the entire serialized
+        block.
+        """
+        try:
+            return self._cached_GetHash
+        except AttributeError:
+            _cached_GetHash = self.get_header().GetHash()
+            object.__setattr__(self, '_cached_GetHash', _cached_GetHash)
+            return _cached_GetHash
+
 class CoreChainParams(object):
     """Define consensus-critical parameters of a given instance of the Bitcoin system"""
     MAX_MONEY = None
@@ -766,6 +883,7 @@ def CheckBlock(block, fCheckPoW = True, fCheckMerkleRoot = True, cur_time=None):
         'CMutableTransaction',
         'CBlockHeader',
         'CBlock',
+        'CMerkleBlock',
         'CoreChainParams',
         'CoreMainParams',
         'CoreTestNetParams',

bitcoin/messages.py

@@ -33,6 +33,7 @@
 from bitcoin.core import *
 from bitcoin.core.serialize import *
 from bitcoin.net import *
+from bitcoin.bloom import CBloomFilter
 import bitcoin
 
 MSG_TX = 1
@@ -386,6 +387,26 @@ def __repr__(self):
         return "msg_block(block=%s)" % (repr(self.block))
 
 
+class msg_merkleblock(MsgSerializable):
+    command = b"merkleblock"
+
+    def __init__(self, protover=PROTO_VERSION):
+        super(msg_merkleblock, self).__init__(protover)
+        self.block = CMerkleBlock()
+
+    @classmethod
+    def msg_deser(cls, f, protover=PROTO_VERSION):
+        c = cls()
+        c.block = CMerkleBlock.stream_deserialize(f)
+        return c
+
+    def msg_ser(self, f):
+        self.block.stream_serialize(f)
+
+    def __repr__(self):
+        return "msg_merkleblock(header=%s)" % (repr(self.block.get_header()))
+
+
 class msg_getaddr(MsgSerializable):
     command = b"getaddr"
 
@@ -485,6 +506,28 @@ def msg_ser(self, f):
     def __repr__(self):
         return "msg_mempool()"
 
+
+class msg_filterload(MsgSerializable):
+    command = b"filterload"
+
+    def __init__(self, protover=PROTO_VERSION, filter=None):
+        super(msg_filterload, self).__init__(protover)
+        self.protover = protover
+        self.filter = filter
+
+    @classmethod
+    def msg_deser(cls, f, protover=PROTO_VERSION):
+        c = cls()
+        c.filter = CBloomFilter.stream_deserialize(f)
+        return c
+
+    def msg_ser(self, f):
+        self.filter.stream_serialize(f)
+
+    def __repr__(self):
+        return "msg_filterload(vData=%i nHashFunctions=%i nTweak=%i nFlags=%i" % (self.filter.vData, self.filter.nHashFunctions, self.filter.nTweak, self.filter.nFlags)
+
+
 msg_classes = [msg_version, msg_verack, msg_addr, msg_alert, msg_inv,
                msg_getdata, msg_notfound, msg_getblocks, msg_getheaders,
                msg_headers, msg_tx, msg_block, msg_getaddr, msg_ping,
@@ -511,10 +554,12 @@ def __repr__(self):
         'msg_headers',
         'msg_tx',
         'msg_block',
+        'msg_merkleblock',
         'msg_getaddr',
         'msg_ping',
         'msg_pong',
         'msg_mempool',
+        'msg_filterload',
         'msg_classes',
         'messagemap',
 )

bitcoin/tests/test_core.py

@@ -120,3 +120,27 @@ def test_calc_merkle_root(self):
         # 99993 four transactions
         block = CBlock.deserialize(x('01000000acda3db591d5c2c63e8c09e7523a5b0581707ef3e3520d6ca180000000000000701179cb9a9e0fe709cc96261b6b943b31362b61dacba94b03f9b71a06cc2eff7d1c1b4d4c86041b75962f880401000000010000000000000000000000000000000000000000000000000000000000000000ffffffff07044c86041b0152ffffffff014034152a01000000434104216220ab283b5e2871c332de670d163fb1b7e509fd67db77997c5568e7c25afd988f19cd5cc5aec6430866ec64b5214826b28e0f7a86458073ff933994b47a5cac0000000001000000042a40ae58b06c3a61ae55dbee05cab546e80c508f71f24ef0cdc9749dac91ea5f000000004a49304602210089c685b37903c4aa62d984929afeaca554d1641f9a668398cd228fb54588f06b0221008a5cfbc5b0a38ba78c4f4341e53272b9cd0e377b2fb740106009b8d7fa693f0b01ffffffff7b999491e30af112b11105cb053bc3633a8a87f44740eb158849a76891ff228b00000000494830450221009a4aa8663ff4017063d2020519f2eade5b4e3e30be69bf9a62b4e6472d1747b2022021ee3b3090b8ce439dbf08a5df31e2dc23d68073ebda45dc573e8a4f74f5cdfc01ffffffffdea82ec2f9e88e0241faa676c13d093030b17c479770c6cc83239436a4327d49000000004a493046022100c29d9de71a34707c52578e355fa0fdc2bb69ce0a957e6b591658a02b1e039d69022100f82c8af79c166a822d305f0832fb800786d831aea419069b3aed97a6edf8f02101fffffffff3e7987da9981c2ae099f97a551783e1b21669ba0bf3aca8fe12896add91a11a0000000049483045022100e332c81781b281a3b35cf75a5a204a2be451746dad8147831255291ebac2604d02205f889a2935270d1bf1ef47db773d68c4d5c6a51bb51f082d3e1c491de63c345601ffffffff0100c817a8040000001976a91420420e56079150b50fb0617dce4c374bd61eccea88ac00000000010000000265a7293b2d69ba51d554cd32ac7586f7fbeaeea06835f26e03a2feab6aec375f000000004a493046022100922361eaafe316003087d355dd3c0ef3d9f44edae661c212a28a91e020408008022100c9b9c84d53d82c0ba9208f695c79eb42a453faea4d19706a8440e1d05e6cff7501fffffffff6971f00725d17c1c531088144b45ed795a307a22d51ca377c6f7f93675bb03a000000008b483045022100d060f2b2f4122edac61a25ea06396fe9135affdabc66d350b5ae1813bc6bf3f302205d8363deef2101fc9f3d528a8b3907e9d29c40772e587dcea12838c574cb80f801410449fce4a25c972a43a6bc67456407a0d4ced782d4cf8c0a35a130d5f65f0561e9f35198349a7c0b4ec79a15fead66bd7642f17cc8c40c5df95f15ac7190c76442ffffffff0200f2052a010000001976a914c3f537bc307c7eda43d86b55695e46047b770ea388ac00cf7b05000000001976a91407bef290008c089a60321b21b1df2d7f2202f40388ac0000000001000000014ab7418ecda2b2531eef0145d4644a4c82a7da1edd285d1aab1ec0595ac06b69000000008c493046022100a796490f89e0ef0326e8460edebff9161da19c36e00c7408608135f72ef0e03e0221009e01ef7bc17cddce8dfda1f1a6d3805c51f9ab2f8f2145793d8e85e0dd6e55300141043e6d26812f24a5a9485c9d40b8712215f0c3a37b0334d76b2c24fcafa587ae5258853b6f49ceeb29cd13ebb76aa79099fad84f516bbba47bd170576b121052f1ffffffff0200a24a04000000001976a9143542e17b6229a25d5b76909f9d28dd6ed9295b2088ac003fab01000000001976a9149cea2b6e3e64ad982c99ebba56a882b9e8a816fe88ac00000000'))
         self.assertEqual(block.calc_merkle_root(), lx('ff2ecc061ab7f9034ba9cbda612b36313b946b1b2696cc09e70f9e9acb791170'))
+
+class Test_CMerkleBlock(unittest.TestCase):
+    def test_serialization(self):
+        initial_serialized = x('0100000082bb869cf3a793432a66e826e05a6fc37469f8efb7421dc880670100000000007f16c5962e8bd963659c793ce370d95f093bc7e367117b3c30c1f8fdd0d9728776381b4d4c86041b554b852907000000043612262624047ee87660be1a707519a443b1c1ce3d248cbfc6c15870f6c5daa2019f5b01d4195ecbc9398fbf3c3b1fa9bb3183301d7a1fb3bd174fcfa40a2b6541ed70551dd7e841883ab8f0b16bf04176b7d1480e4f0af9f3d4c3595768d06820d2a7bc994987302e5b1ac80fc425fe25f8b63169ea78e68fbaaefa59379bbf011d')
+        block = CMerkleBlock.deserialize(initial_serialized)
+        self.assertEqual(block.nTX, 7)
+        self.assertEqual(len(block.vHashes), 4)
+        self.assertEqual(len(block.vFlags), 8)
+        serialized = block.serialize()
+        self.assertEqual(Hash(serialized[0:80]), Hash(initial_serialized[:80]))
+        self.assertEqual(serialized, initial_serialized)
+
+    def test_GetHash(self):
+        block = CMerkleBlock.deserialize(x('0100000082bb869cf3a793432a66e826e05a6fc37469f8efb7421dc880670100000000007f16c5962e8bd963659c793ce370d95f093bc7e367117b3c30c1f8fdd0d9728776381b4d4c86041b554b852907000000043612262624047ee87660be1a707519a443b1c1ce3d248cbfc6c15870f6c5daa2019f5b01d4195ecbc9398fbf3c3b1fa9bb3183301d7a1fb3bd174fcfa40a2b6541ed70551dd7e841883ab8f0b16bf04176b7d1480e4f0af9f3d4c3595768d06820d2a7bc994987302e5b1ac80fc425fe25f8b63169ea78e68fbaaefa59379bbf011d'))
+        self.assertEqual(block.GetHash(), lx('000000000000b731f2eef9e8c63173adfb07e41bd53eb0ef0a6b720d6cb6dea4'))
+
+    def test_extract_hashes(self):
+        initial_serialized = x('0100000082bb869cf3a793432a66e826e05a6fc37469f8efb7421dc880670100000000007f16c5962e8bd963659c793ce370d95f093bc7e367117b3c30c1f8fdd0d9728776381b4d4c86041b554b852907000000043612262624047ee87660be1a707519a443b1c1ce3d248cbfc6c15870f6c5daa2019f5b01d4195ecbc9398fbf3c3b1fa9bb3183301d7a1fb3bd174fcfa40a2b6541ed70551dd7e841883ab8f0b16bf04176b7d1480e4f0af9f3d4c3595768d06820d2a7bc994987302e5b1ac80fc425fe25f8b63169ea78e68fbaaefa59379bbf011d')
+        block = CMerkleBlock.deserialize(initial_serialized)
+        matches = block.get_matched_txs()
+        self.assertEqual(len(matches), 1)
+        self.assertEqual(matches[0], x("019f5b01d4195ecbc9398fbf3c3b1fa9bb3183301d7a1fb3bd174fcfa40a2b65"))
+
+

bitcoin/tests/test_messages.py

@@ -13,8 +13,8 @@
 
 from bitcoin.messages import msg_version, msg_verack, msg_addr, msg_alert, \
     msg_inv, msg_getdata, msg_getblocks, msg_getheaders, msg_headers, msg_tx, \
-    msg_block, msg_getaddr, msg_ping, msg_pong, msg_mempool, MsgSerializable, \
-    msg_notfound, msg_reject
+    msg_block, msg_merkleblock, msg_getaddr, msg_ping, msg_pong, msg_mempool, msg_filterload, \
+    MsgSerializable, msg_notfound, msg_reject
 
 import sys
 if sys.version > '3':
@@ -92,6 +92,11 @@ def test_serialization(self):
         super(Test_msg_block, self).serialization_test(msg_block)
 
 
+class Test_msg_merkleblock(MessageTestCase):
+    def test_serialization(self):
+        super(Test_msg_merkleblock, self).serialization_test(msg_merkleblock)
+
+
 class Test_msg_getaddr(MessageTestCase):
     def test_serialization(self):
         super(Test_msg_getaddr, self).serialization_test(msg_getaddr)
@@ -117,6 +122,11 @@ def test_serialization(self):
         super(Test_msg_mempool, self).serialization_test(msg_mempool)
 
 
+class Test_msg_filterload(MessageTestCase):
+    def test_serialization(self):
+        super(Test_msg_filterload, self).serialization_test(msg_filterload)
+
+
 class Test_messages(unittest.TestCase):
     verackbytes = b'\xf9\xbe\xb4\xd9verack\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00]\xf6\xe0\xe2'