Rework message serialization

Add new class 'MsgSerializable' subclassing the already existing
'Serializable' class.

Remove messagemap, the message strings are now a static attribute of
every message class.

Remove message_read, replaced by the classmethod
MsgSerializeable.stream_deserialize()

Remove message_to_str, replaced by MsgSerializeable.to_bytes()

Make classes in bitcoin.net subclasses of Serializable

Add tests for messages and net

Add examples/msg-serializable.py

Add test_suite to setup.py

Author: Flowdalic

.gitignore

@@ -3,3 +3,5 @@
 
 local*.cfg
 
+build/
+python_bitcoinlib.egg-info/

bitcoin/core/__init__.py

@@ -23,9 +23,12 @@
 MAX_BLOCK_SIZE = 1000000
 MAX_BLOCK_SIGOPS = MAX_BLOCK_SIZE/50
 
+BIP0031_VERSION = 60000
 PROTO_VERSION = 60002
 MIN_PROTO_VERSION = 209
 
+CADDR_TIME_VERSION = 31402
+
 def MoneyRange(nValue):
         return 0<= nValue <= MAX_MONEY
 
@@ -69,7 +72,7 @@ def b2lx(b):
 
 def str_money_value(value):
     """Convert an integer money value to a fixed point string"""
-    r = '%i.%08i' % (value // 100000000, value % 100000000)
+    r = '%i.%08i' % (value // COIN, value % COIN)
     r = r.rstrip('0')
     if r[-1] == '.':
         r += '0'
@@ -224,9 +227,9 @@ def __repr__(self):
 
 class CBlockHeader(Serializable):
     """A block header"""
-    __slots__ = ['nVersion', 'hashPrevBlock', 'hashMerkleRoot', 'nTime', 'nBits', 'nBits']
+    __slots__ = ['nVersion', 'hashPrevBlock', 'hashMerkleRoot', 'nTime', 'nBits', 'nNonce']
 
-    def __init__(self, nVersion=2, hashPrevBlock=None, hashMerkleRoot=None, nTime=None, nBits=None, nNonce=None):
+    def __init__(self, nVersion=2, hashPrevBlock=num_null_bytes(32), hashMerkleRoot=num_null_bytes(32), nTime=0, nBits=0, nNonce=0):
         self.nVersion = nVersion
         assert len(hashPrevBlock) == 32
         self.hashPrevBlock = hashPrevBlock
@@ -279,7 +282,7 @@ class CBlock(CBlockHeader):
     """A block including all transactions in it"""
     __slots__ = ['vtx']
 
-    def __init__(self, nVersion=2, hashPrevBlock=None, hashMerkleRoot=None, nTime=None, nBits=None, nNonce=None, vtx=None):
+    def __init__(self, nVersion=2, hashPrevBlock=num_null_bytes(32), hashMerkleRoot=num_null_bytes(32), nTime=0, nBits=0, nNonce=0, vtx=None):
         super(CBlock, self).__init__(nVersion, hashPrevBlock, hashMerkleRoot, nTime, nBits, nNonce)
         if not vtx:
             vtx = []

bitcoin/core/serialize.py

@@ -18,26 +18,30 @@
 
 # Py3 compatibility
 import sys
-bchr = chr
-bord = ord
+
 if sys.version > '3':
     bchr = lambda x: bytes([x])
     bord = lambda x: x[0]
     from io import BytesIO
 else:
+    bchr = chr
+    bord = ord
     from cStringIO import StringIO as BytesIO
 
 MAX_SIZE = 0x02000000
 
+
 class SerializationError(Exception):
     """Base class for serialization errors"""
 
+
 class SerializationTruncationError(Exception):
     """Serialized data was truncated
 
     Thrown by deserialize() and stream_deserialize()
     """
 
+
 def ser_read(f, n):
     """Read from a stream safely
 
@@ -49,9 +53,17 @@ def ser_read(f, n):
         raise SerializationError('Asked to read 0x%x bytes; MAX_SIZE exceeded')
     r = f.read(n)
     if len(r) < n:
-        raise SerializationTruncationError()
+        raise SerializationTruncationError('Asked to read %i bytes, but only got %i' % (n, len(r)))
     return r
 
+
+def num_null_bytes(num):
+    res = b''
+    for i in range(num):
+        res += b'\x00'
+    return res
+
+
 class Serializable(object):
     """Base class for serializable objects"""
     def stream_serialize(self, f):
@@ -86,6 +98,7 @@ def __ne__(self, other):
     def __hash__(self):
         return hash(self.serialize())
 
+
 class Serializer(object):
     """Base class for object serializers"""
     def __new__(cls):
@@ -108,6 +121,7 @@ def serialize(cls, obj):
     def deserialize(cls, buf):
         return cls.stream_deserialize(BytesIO(buf))
 
+
 class VarIntSerializer(Serializer):
     """Serialization of variable length ints"""
     @classmethod
@@ -138,6 +152,7 @@ def stream_deserialize(cls, f):
         else:
             return struct.unpack(b'<Q', ser_read(f, 8))[0]
 
+
 class BytesSerializer(Serializer):
     """Serialization of bytes instances"""
     @classmethod
@@ -150,6 +165,7 @@ def stream_deserialize(cls, f):
         l = VarIntSerializer.stream_deserialize(f)
         return ser_read(f, l)
 
+
 class VectorSerializer(Serializer):
     """Base class for serializers of object vectors"""
     @classmethod
@@ -166,23 +182,55 @@ def stream_deserialize(cls, inner_cls, f):
             r.append(inner_cls.stream_deserialize(f))
         return r
 
+
 class uint256VectorSerializer(Serializer):
     """Serialize vectors of uint256"""
     @classmethod
-    def stream_serialize(cls, inner_cls, objs, f):
-        VarIntSerializer.stream_serialize(len(objs), f)
-        for obj in objs:
-            assert len(obj) == 32
-            f.write(obj)
+    def stream_serialize(cls, uints, f):
+        VarIntSerializer.stream_serialize(len(uints), f)
+        for uint in uints:
+            assert len(uint) == 32
+            f.write(uint)
 
     @classmethod
-    def stream_deserialize(cls, inner_cls, f):
+    def stream_deserialize(cls, f):
         n = VarIntSerializer.stream_deserialize(f)
         r = []
         for i in range(n):
             r.append(ser_read(f, 32))
         return r
 
+
+class intVectorSerialzer(Serializer):
+    @classmethod
+    def stream_serialize(cls, ints, f):
+        l = len(ints)
+        VarIntSerializer.stream_serialize(l, f)
+        for i in ints:
+            f.write(struct.pack(b"<i", i))
+
+    @classmethod
+    def stream_deserialize(cls, f):
+        l = VarIntSerializer.stream_deserialize(f)
+        ints = []
+        for i in range(l):
+            ints.append(struct.unpack(b"<i", ser_read(f, 4)))
+
+
+class VarStringSerializer(Serializer):
+    """Serialize variable length strings"""
+    @classmethod
+    def stream_serialize(cls, s, f):
+        l = len(s)
+        VarIntSerializer.stream_serialize(l, f)
+        f.write(s)
+
+    @classmethod
+    def stream_deserialize(cls, f):
+        l = VarIntSerializer.stream_deserialize(f)
+        return ser_read(f, l)
+
+
 def uint256_from_str(s):
     """Convert bytes to uint256"""
     r = 0
@@ -191,6 +239,7 @@ def uint256_from_str(s):
         r += t[i] << (i * 32)
     return r
 
+
 def uint256_from_compact(c):
     """Convert compact encoding to uint256
 
@@ -200,44 +249,17 @@ def uint256_from_compact(c):
     v = (c & 0xFFFFFF) << (8 * (nbytes - 3))
     return v
 
+
 def uint256_to_shortstr(u):
     s = "%064x" % (u,)
     return s[:16]
 
-def deser_int_vector(f):
-    """Deserialize a vector of ints"""
-    nit = struct.unpack(b"<B", f.read(1))[0]
-    if nit == 253:
-        nit = struct.unpack(b"<H", f.read(2))[0]
-    elif nit == 254:
-        nit = struct.unpack(b"<I", f.read(4))[0]
-    elif nit == 255:
-        nit = struct.unpack(b"<Q", f.read(8))[0]
-    r = []
-    for i in range(nit):
-        t = struct.unpack(b"<i", f.read(4))[0]
-        r.append(t)
-    return r
-
-def ser_int_vector(l):
-    """Serialize a vector of ints"""
-    r = b""
-    if len(l) < 253:
-        r = bchr(len(l))
-    elif len(s) < 0x10000:
-        r = bchr(253) + struct.pack(b"<H", len(l))
-    elif len(s) < 0x100000000:
-        r = bchr(254) + struct.pack(b"<I", len(l))
-    else:
-        r = bchr(255) + struct.pack(b"<Q", len(l))
-    for i in l:
-        r += struct.pack(b"<i", i)
-    return r
 
 def Hash(msg):
     """SHA256^2)(msg) -> bytes"""
     return hashlib.sha256(hashlib.sha256(msg).digest()).digest()
 
+
 def Hash160(msg):
     """RIPEME160(SHA256(msg)) -> bytes"""
     h = hashlib.new('ripemd160')